Lines Matching refs:dhp
88 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ argument
89 { mutex_enter(&dhp->dh_lock); }
91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ argument
92 { mutex_exit(&dhp->dh_lock); }
151 #define dhp_is_devmem(dhp) \ argument
152 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
154 #define dhp_is_pmem(dhp) \ argument
155 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
157 #define dhp_is_kpmem(dhp) \ argument
158 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
245 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
248 static void free_devmap_handle(devmap_handle_t *dhp);
250 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
253 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
255 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
257 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
259 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
262 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
265 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
364 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data; in segdev_create() local
387 sdp->devmap_data = dhp; in segdev_create()
389 sdp->devmap_data = dhp = NULL; in segdev_create()
410 while (dhp != NULL) { in segdev_create()
411 dhp->dh_seg = seg; in segdev_create()
412 dhp = dhp->dh_next; in segdev_create()
463 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; in segdev_dup() local
468 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg); in segdev_dup()
471 (void *)dhp, (void *)seg)); in segdev_dup()
513 if (dhp != NULL) { in segdev_dup()
514 ret = devmap_handle_dup(dhp, in segdev_dup()
519 ret, (void *)dhp, (void *)seg); in segdev_dup()
522 ret, (void *)dhp, (void *)seg)); in segdev_dup()
540 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp, in devmap_handle_dup() argument
547 while (dhp != NULL) { in devmap_handle_dup()
551 HOLD_DHP_LOCK(dhp); in devmap_handle_dup()
552 bcopy(dhp, newdhp, sizeof (devmap_handle_t)); in devmap_handle_dup()
553 RELE_DHP_LOCK(dhp); in devmap_handle_dup()
564 if (dhp->dh_softlock != NULL) in devmap_handle_dup()
568 if (dhp->dh_ctx != NULL) in devmap_handle_dup()
587 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp, in devmap_handle_dup()
603 (void *)dhp)); in devmap_handle_dup()
609 dhp = dhp->dh_next; in devmap_handle_dup()
629 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; in segdev_unmap() local
640 (void *)dhp, (void *)seg, (void *)addr, len); in segdev_unmap()
643 (void *)dhp, (void *)seg, (void *)addr, len)); in segdev_unmap()
669 if (dhp != NULL) { in segdev_unmap()
675 dhpp = tdhp = devmap_find_handle(dhp, addr); in segdev_unmap()
752 if (dhp != NULL) in segdev_unmap()
753 devmap_handle_unmap_head(dhp, len); in segdev_unmap()
784 if (dhp != NULL) in segdev_unmap()
785 devmap_handle_unmap_tail(dhp, addr); in segdev_unmap()
854 if (dhp == NULL) { in segdev_unmap()
858 while (dhp != NULL) { in segdev_unmap()
859 callbackops = &dhp->dh_callbackops; in segdev_unmap()
861 "segdev_unmap: dhp=%p addr=%p", dhp, addr); in segdev_unmap()
863 (void *)dhp, (void *)addr, in segdev_unmap()
864 (void *)dhp->dh_uvaddr, dhp->dh_len)); in segdev_unmap()
866 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
867 dhpp = dhp->dh_next; in segdev_unmap()
868 dhp->dh_next = NULL; in segdev_unmap()
869 dhp = dhpp; in segdev_unmap()
870 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
871 dhp = dhp->dh_next; in segdev_unmap()
872 } else if (addr > dhp->dh_uvaddr && in segdev_unmap()
873 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
880 HOLD_DHP_LOCK(dhp); in segdev_unmap()
881 bcopy(dhp, newdhp, sizeof (devmap_handle_t)); in segdev_unmap()
882 RELE_DHP_LOCK(dhp); in segdev_unmap()
884 newdhp->dh_next = dhp->dh_next; in segdev_unmap()
885 if (dhp->dh_softlock != NULL) in segdev_unmap()
889 if (dhp->dh_ctx != NULL) in segdev_unmap()
897 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in segdev_unmap()
898 off, len, dhp, &dhp->dh_pvtp, in segdev_unmap()
900 mlen = len + (addr - dhp->dh_uvaddr); in segdev_unmap()
904 dhp->dh_len = addr - dhp->dh_uvaddr; in segdev_unmap()
905 dhpp = dhp->dh_next; in segdev_unmap()
906 dhp->dh_next = NULL; in segdev_unmap()
907 dhp = dhpp; in segdev_unmap()
908 } else if ((addr > dhp->dh_uvaddr) && in segdev_unmap()
909 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) { in segdev_unmap()
910 mlen = dhp->dh_len + dhp->dh_uvaddr - addr; in segdev_unmap()
915 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in segdev_unmap()
916 off, mlen, (devmap_cookie_t *)dhp, in segdev_unmap()
917 &dhp->dh_pvtp, NULL, NULL); in segdev_unmap()
919 dhp->dh_len = addr - dhp->dh_uvaddr; in segdev_unmap()
920 dhpp = dhp->dh_next; in segdev_unmap()
921 dhp->dh_next = NULL; in segdev_unmap()
922 dhp = dhpp; in segdev_unmap()
923 nsdp->devmap_data = dhp; in segdev_unmap()
924 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
928 dhp->dh_seg = nseg; in segdev_unmap()
929 nsdp->devmap_data = dhp; in segdev_unmap()
930 dhp = devmap_handle_unmap(dhp); in segdev_unmap()
931 nsdp->devmap_data = dhp; /* XX redundant? */ in segdev_unmap()
932 } else if (((addr + len) > dhp->dh_uvaddr) && in segdev_unmap()
933 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) { in segdev_unmap()
934 mlen = addr + len - dhp->dh_uvaddr; in segdev_unmap()
936 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in segdev_unmap()
937 dhp->dh_uoff, mlen, NULL, in segdev_unmap()
938 NULL, dhp, &dhp->dh_pvtp); in segdev_unmap()
939 devmap_handle_reduce_len(dhp, mlen); in segdev_unmap()
940 nsdp->devmap_data = dhp; in segdev_unmap()
941 dhp->dh_seg = nseg; in segdev_unmap()
942 dhp = dhp->dh_next; in segdev_unmap()
944 dhp->dh_seg = nseg; in segdev_unmap()
945 dhp = dhp->dh_next; in segdev_unmap()
958 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len) in devmap_handle_reduce_len() argument
965 ASSERT(len < dhp->dh_len); in devmap_handle_reduce_len()
970 dhp->dh_len -= len; in devmap_handle_reduce_len()
971 dhp->dh_uoff += (offset_t)len; in devmap_handle_reduce_len()
972 dhp->dh_roff += (offset_t)len; in devmap_handle_reduce_len()
973 dhp->dh_uvaddr += len; in devmap_handle_reduce_len()
975 HOLD_DHP_LOCK(dhp); in devmap_handle_reduce_len()
976 cp = dhp->dh_cookie; in devmap_handle_reduce_len()
977 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) { in devmap_handle_reduce_len()
979 dhp->dh_pfn += btop(len); in devmap_handle_reduce_len()
981 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; in devmap_handle_reduce_len()
982 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && in devmap_handle_reduce_len()
983 dhp->dh_roff < ptob(pcp->dp_npages)); in devmap_handle_reduce_len()
985 ASSERT(dhp->dh_roff < cp->size); in devmap_handle_reduce_len()
986 ASSERT(dhp->dh_cvaddr >= cp->cvaddr && in devmap_handle_reduce_len()
987 dhp->dh_cvaddr < (cp->cvaddr + cp->size)); in devmap_handle_reduce_len()
988 ASSERT((dhp->dh_cvaddr + len) <= in devmap_handle_reduce_len()
991 dhp->dh_cvaddr += len; in devmap_handle_reduce_len()
995 RELE_DHP_LOCK(dhp); in devmap_handle_reduce_len()
1003 devmap_handle_unmap(devmap_handle_t *dhp) in devmap_handle_unmap() argument
1005 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops; in devmap_handle_unmap()
1006 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data; in devmap_handle_unmap()
1009 ASSERT(dhp != NULL); in devmap_handle_unmap()
1016 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff, in devmap_handle_unmap()
1017 dhp->dh_len, NULL, NULL, NULL, NULL); in devmap_handle_unmap()
1020 if (dhpp == dhp) { /* releasing first dhp, change sdp data */ in devmap_handle_unmap()
1021 sdp->devmap_data = dhp->dh_next; in devmap_handle_unmap()
1023 while (dhpp->dh_next != dhp) { in devmap_handle_unmap()
1026 dhpp->dh_next = dhp->dh_next; in devmap_handle_unmap()
1028 dhpp = dhp->dh_next; /* return value is next dhp in chain */ in devmap_handle_unmap()
1030 if (dhp->dh_softlock != NULL) in devmap_handle_unmap()
1031 devmap_softlock_rele(dhp); in devmap_handle_unmap()
1033 if (dhp->dh_ctx != NULL) in devmap_handle_unmap()
1034 devmap_ctx_rele(dhp); in devmap_handle_unmap()
1036 if (dhp->dh_flags & DEVMAP_LOCK_INITED) { in devmap_handle_unmap()
1037 mutex_destroy(&dhp->dh_lock); in devmap_handle_unmap()
1039 kmem_free(dhp, sizeof (devmap_handle_t)); in devmap_handle_unmap()
1049 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len) in devmap_handle_unmap_head() argument
1056 while (len >= dhp->dh_len) { in devmap_handle_unmap_head()
1057 len -= dhp->dh_len; in devmap_handle_unmap_head()
1058 dhp = devmap_handle_unmap(dhp); in devmap_handle_unmap_head()
1061 callbackops = &dhp->dh_callbackops; in devmap_handle_unmap_head()
1068 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in devmap_handle_unmap_head()
1069 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp); in devmap_handle_unmap_head()
1070 devmap_handle_reduce_len(dhp, len); in devmap_handle_unmap_head()
1080 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr) in devmap_handle_unmap_tail() argument
1082 register struct seg *seg = dhp->dh_seg; in devmap_handle_unmap_tail()
1091 maplen = (size_t)(addr - dhp->dh_uvaddr); in devmap_handle_unmap_tail()
1127 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; in segdev_free() local
1130 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg); in segdev_free()
1132 (void *)dhp, (void *)seg)); in segdev_free()
1140 while (dhp != NULL) in segdev_free()
1141 dhp = devmap_handle_unmap(dhp); in segdev_free()
1152 free_devmap_handle(devmap_handle_t *dhp) in free_devmap_handle() argument
1159 while (dhp != NULL) { in free_devmap_handle()
1160 dhpp = dhp->dh_next; in free_devmap_handle()
1161 if (dhp->dh_flags & DEVMAP_LOCK_INITED) { in free_devmap_handle()
1162 mutex_destroy(&dhp->dh_lock); in free_devmap_handle()
1165 if (dhp->dh_softlock != NULL) in free_devmap_handle()
1166 devmap_softlock_rele(dhp); in free_devmap_handle()
1168 if (dhp->dh_ctx != NULL) in free_devmap_handle()
1169 devmap_ctx_rele(dhp); in free_devmap_handle()
1171 kmem_free(dhp, sizeof (devmap_handle_t)); in free_devmap_handle()
1172 dhp = dhpp; in free_devmap_handle()
1341 devmap_handle_t *dhp; in segdev_softunlock() local
1346 dhp = devmap_find_handle(dhp_head, addr); in segdev_softunlock()
1347 ASSERT(dhp != NULL); in segdev_softunlock()
1349 off = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_softunlock()
1351 mlen = MIN(tlen, (dhp->dh_len - off)); in segdev_softunlock()
1356 if (dhp_is_kpmem(dhp)) { in segdev_softunlock()
1358 (struct ddi_umem_cookie *)dhp->dh_cookie, in segdev_softunlock()
1365 if (dhp->dh_callbackops.devmap_access != NULL) { in segdev_softunlock()
1366 devmap_softlock_exit(dhp->dh_softlock, in segdev_softunlock()
1371 dhp = dhp->dh_next; in segdev_softunlock()
1414 devmap_handle_t *dhp) /* devmap handle if any for this page */ in segdev_faultpage() argument
1424 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr); in segdev_faultpage()
1426 (void *)dhp, (void *)seg, (void *)addr)); in segdev_faultpage()
1472 if (dhp == NULL) { in segdev_faultpage()
1484 ASSERT(dhp == devmap_find_handle( in segdev_faultpage()
1487 off = addr - dhp->dh_uvaddr; in segdev_faultpage()
1494 cp = dhp->dh_cookie; in segdev_faultpage()
1495 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { in segdev_faultpage()
1498 pfnum = dhp->dh_pfn + btop(off); in segdev_faultpage()
1500 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; in segdev_faultpage()
1501 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && in segdev_faultpage()
1502 dhp->dh_roff < ptob(pcp->dp_npages)); in segdev_faultpage()
1504 pcp->dp_pparray[btop(off + dhp->dh_roff)]); in segdev_faultpage()
1506 ASSERT(dhp->dh_roff < cp->size); in segdev_faultpage()
1507 ASSERT(dhp->dh_cvaddr >= cp->cvaddr && in segdev_faultpage()
1508 dhp->dh_cvaddr < (cp->cvaddr + cp->size)); in segdev_faultpage()
1509 ASSERT((dhp->dh_cvaddr + off) <= in segdev_faultpage()
1511 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <= in segdev_faultpage()
1517 ASSERT((dhp->dh_roff & in segdev_faultpage()
1521 dhp->dh_roff)]); in segdev_faultpage()
1538 dhp->dh_cvaddr + off); in segdev_faultpage()
1545 prot |= dhp->dh_hat_attr; in segdev_faultpage()
1558 if (pf_is_memory(pfnum) || (dhp != NULL)) { in segdev_faultpage()
1605 devmap_handle_t *dhp; in segdev_fault() local
1627 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) in segdev_fault()
1699 devmap_handle_t *dhpp = dhp; in segdev_fault()
1775 off = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_fault()
1790 mlen = MIN(len, (dhp->dh_len - off)); in segdev_fault()
1792 HOLD_DHP_LOCK(dhp); in segdev_fault()
1798 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) { in segdev_fault()
1799 devmap_get_large_pgsize(dhp, mlen, maddr, in segdev_fault()
1807 if (dhp->dh_callbackops.devmap_access != NULL) { in segdev_fault()
1819 RELE_DHP_LOCK(dhp); in segdev_fault()
1821 err = (*dhp->dh_callbackops.devmap_access)( in segdev_fault()
1822 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw); in segdev_fault()
1829 type, rw, dhp); in segdev_fault()
1831 RELE_DHP_LOCK(dhp); in segdev_fault()
1876 dhp = dhp->dh_next; in segdev_fault()
1879 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr); in segdev_fault()
1907 devmap_handle_t *dhp) /* devmap handle */ in segdev_faultpages() argument
1917 (void *)dhp, (void *)seg, (void *)addr, len); in segdev_faultpages()
1920 (void *)dhp, (void *)seg, (void *)addr, len)); in segdev_faultpages()
1945 if ((dhp != NULL) && dhp_is_kpmem(dhp)) { in segdev_faultpages()
1946 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie; in segdev_faultpages()
1991 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) { in segdev_faultpages()
1995 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { in segdev_faultpages()
2007 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr)); in segdev_faultpages()
2010 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr, in segdev_faultpages()
2025 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) { in segdev_faultpages()
2069 register devmap_handle_t *dhp; in segdev_setprot() local
2093 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) in segdev_setprot()
2099 off = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_setprot()
2101 while (dhp) { in segdev_setprot()
2102 if ((dhp->dh_maxprot & prot) != prot) in segdev_setprot()
2105 if (mlen > (dhp->dh_len - off)) { in segdev_setprot()
2106 mlen -= dhp->dh_len - off; in segdev_setprot()
2107 dhp = dhp->dh_next; in segdev_setprot()
2152 dhp = tdhp = devmap_find_handle(dhp_head, addr); in segdev_setprot()
2164 soff = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_setprot()
2166 mlen = MIN(slen, (dhp->dh_len - soff)); in segdev_setprot()
2167 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr, in segdev_setprot()
2168 dhp->dh_len, HAT_UNLOAD); in segdev_setprot()
2169 dhp = dhp->dh_next; in segdev_setprot()
2497 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr, in devmap_device() argument
2510 (void *)dhp, (void *)addr, off, len); in devmap_device()
2513 (void *)dhp, (void *)addr, off, len)); in devmap_device()
2519 rdhp = maxdhp = dhp; in devmap_device()
2524 maxprot |= dhp->dh_maxprot; in devmap_device()
2526 offset = maxdhp->dh_uoff - dhp->dh_uoff; in devmap_device()
2568 dev_a.dev = dhp->dh_dev; in devmap_device()
2575 dev_a.prot = dhp->dh_prot; in devmap_device()
2581 dev_a.devmap_data = (void *)dhp; in devmap_device()
2593 register devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_do_ctxmgt() local
2604 (void *)dhp, off, len); in devmap_do_ctxmgt()
2606 (void *)dhp, off, len)); in devmap_do_ctxmgt()
2611 devctx = dhp->dh_ctx; in devmap_do_ctxmgt()
2624 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) { in devmap_do_ctxmgt()
2627 devctx, dhp); in devmap_do_ctxmgt()
2640 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw); in devmap_do_ctxmgt()
2652 ret, dhp, devctx); in devmap_do_ctxmgt()
2654 ret, (void *)dhp)); in devmap_do_ctxmgt()
2669 if (dhp->dh_timeout_length > 0) { in devmap_do_ctxmgt()
2673 devctx, dhp->dh_timeout_length); in devmap_do_ctxmgt()
2704 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len, in devmap_roundup() argument
2716 (void *)dhp, offset, len); in devmap_roundup()
2718 (void *)dhp, offset, len)); in devmap_roundup()
2728 base = (ulong_t)ptob(dhp->dh_pfn); in devmap_roundup()
2729 for (level = dhp->dh_mmulevel; level >= 0; level--) { in devmap_roundup()
2732 uvaddr = dhp->dh_uvaddr + (poff - base); in devmap_roundup()
2734 ((poff + pg) <= (base + dhp->dh_len)) && in devmap_roundup()
2741 base, poff, dhp); in devmap_roundup()
2743 base, poff, dhp->dh_pfn)); in devmap_roundup()
2749 *opfn = dhp->dh_pfn + btop(poff - base); in devmap_roundup()
2757 (void *)dhp, level, rlen, pagesize, opfn); in devmap_roundup()
2760 (void *)dhp, level, rlen, *pagesize, *opfn)); in devmap_roundup()
2771 devmap_handle_t *dhp; in devmap_find_handle() local
2776 dhp = dhp_head; in devmap_find_handle()
2777 while (dhp) { in devmap_find_handle()
2778 if (addr >= dhp->dh_uvaddr && in devmap_find_handle()
2779 addr < (dhp->dh_uvaddr + dhp->dh_len)) in devmap_find_handle()
2780 return (dhp); in devmap_find_handle()
2781 dhp = dhp->dh_next; in devmap_find_handle()
2796 register devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_unload() local
2803 (void *)dhp, offset, len); in devmap_unload()
2805 (void *)dhp, offset, len)); in devmap_unload()
2807 soff = (ssize_t)(offset - dhp->dh_uoff); in devmap_unload()
2809 if (soff < 0 || soff >= dhp->dh_len) in devmap_unload()
2825 size = dhp->dh_len - soff; in devmap_unload()
2828 if ((soff + size) > dhp->dh_len) in devmap_unload()
2836 addr = (caddr_t)(soff + dhp->dh_uvaddr); in devmap_unload()
2842 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) { in devmap_unload()
2843 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, in devmap_unload()
2844 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); in devmap_unload()
2846 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size, in devmap_unload()
2857 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr, in devmap_get_large_pgsize() argument
2873 ASSERT(dhp_is_devmem(dhp)); in devmap_get_large_pgsize()
2874 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)); in devmap_get_large_pgsize()
2877 off = (ulong_t)(addr - dhp->dh_uvaddr); in devmap_get_large_pgsize()
2884 len = devmap_roundup(dhp, off, len, &pfn, &pgsize); in devmap_get_large_pgsize()
2887 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn); in devmap_get_large_pgsize()
2892 off = ptob(pfn - dhp->dh_pfn) + pgsize; in devmap_get_large_pgsize()
2942 devmap_softlock_rele(devmap_handle_t *dhp) in devmap_softlock_rele() argument
2944 struct devmap_softlock *slock = dhp->dh_softlock; in devmap_softlock_rele()
2993 devmap_ctx_rele(devmap_handle_t *dhp) in devmap_ctx_rele() argument
2995 struct devmap_ctx *devctx = dhp->dh_ctx; in devmap_ctx_rele()
3064 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_load() local
3065 struct as *asp = dhp->dh_seg->s_as; in devmap_load()
3073 (void *)dhp, offset, len); in devmap_load()
3076 (void *)dhp, offset, len)); in devmap_load()
3086 soff = (ssize_t)(offset - dhp->dh_uoff); in devmap_load()
3088 if (soff < 0 || soff >= dhp->dh_len) in devmap_load()
3104 size = dhp->dh_len - soff; in devmap_load()
3107 if ((soff + size) > dhp->dh_len) in devmap_load()
3115 addr = (caddr_t)(soff + dhp->dh_uvaddr); in devmap_load()
3117 HOLD_DHP_LOCK(dhp); in devmap_load()
3119 dhp->dh_seg, addr, size, type, rw, dhp); in devmap_load()
3120 RELE_DHP_LOCK(dhp); in devmap_load()
3128 register devmap_handle_t *dhp; in devmap_setup() local
3178 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP); in devmap_setup()
3181 dhp_prev->dh_next = dhp; in devmap_setup()
3183 dhp_head = dhp; in devmap_setup()
3184 dhp_prev = dhp; in devmap_setup()
3186 dhp->dh_prot = prot; in devmap_setup()
3187 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot; in devmap_setup()
3188 dhp->dh_dev = dev; in devmap_setup()
3189 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE; in devmap_setup()
3190 dhp->dh_uoff = map_off; in devmap_setup()
3198 if ((ret = cdev_devmap(dev, dhp, map_off, in devmap_setup()
3209 callbackops = &dhp->dh_callbackops; in devmap_setup()
3227 dhp->dh_softlock = devmap_softlock_init(dev, in devmap_setup()
3229 dhp->dh_ctx = devmap_ctxinit(dev, in devmap_setup()
3242 slock = dhp->dh_softlock; in devmap_setup()
3243 if (slock != dhp->dh_softlock) { in devmap_setup()
3271 dhp = dhp_head; in devmap_setup()
3274 while (dhp != NULL) { in devmap_setup()
3275 callbackops = &dhp->dh_callbackops; in devmap_setup()
3276 dhp->dh_uvaddr = addr; in devmap_setup()
3277 dhp_curr = dhp; in devmap_setup()
3279 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp, in devmap_setup()
3281 dhp->dh_len, &dhp->dh_pvtp); in devmap_setup()
3289 dhp = dhp_head; in devmap_setup()
3291 while (dhp != dhp_curr) { in devmap_setup()
3292 callbackops = &dhp->dh_callbackops; in devmap_setup()
3295 dhp, dhp->dh_pvtp, in devmap_setup()
3296 map_off, dhp->dh_len, in devmap_setup()
3299 map_off += dhp->dh_len; in devmap_setup()
3300 dhp = dhp->dh_next; in devmap_setup()
3308 map_off += dhp->dh_len; in devmap_setup()
3309 addr += dhp->dh_len; in devmap_setup()
3310 dhp = dhp->dh_next; in devmap_setup()
3334 devmap_devmem_large_page_setup(devmap_handle_t *dhp) in devmap_devmem_large_page_setup() argument
3336 ASSERT(dhp_is_devmem(dhp)); in devmap_devmem_large_page_setup()
3337 dhp->dh_mmulevel = 0; in devmap_devmem_large_page_setup()
3348 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) { in devmap_devmem_large_page_setup()
3352 base = (ulong_t)ptob(dhp->dh_pfn); in devmap_devmem_large_page_setup()
3355 if ((dhp->dh_len < pgsize) || in devmap_devmem_large_page_setup()
3356 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr, in devmap_devmem_large_page_setup()
3361 dhp->dh_mmulevel = level - 1; in devmap_devmem_large_page_setup()
3363 if (dhp->dh_mmulevel > 0) { in devmap_devmem_large_page_setup()
3364 dhp->dh_flags |= DEVMAP_FLAG_LARGE; in devmap_devmem_large_page_setup()
3366 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; in devmap_devmem_large_page_setup()
3379 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_devmem_setup() local
3387 (void *)dhp, roff, rnumber, (uint_t)len); in devmap_devmem_setup()
3389 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); in devmap_devmem_setup()
3394 if (dhp->dh_flags & DEVMAP_SETUP_DONE) in devmap_devmem_setup()
3397 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_devmem_setup()
3411 dhp->dh_pfn = PFN_INVALID; in devmap_devmem_setup()
3429 mr.map_prot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_setup()
3439 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn); in devmap_devmem_setup()
3440 dhp->dh_hat_attr = hp->ah_hat_flags; in devmap_devmem_setup()
3447 ASSERT(!pf_is_memory(dhp->dh_pfn)); in devmap_devmem_setup()
3450 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); in devmap_devmem_setup()
3451 dhp->dh_len = ptob(btopr(len)); in devmap_devmem_setup()
3453 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; in devmap_devmem_setup()
3454 dhp->dh_roff = ptob(btop(roff)); in devmap_devmem_setup()
3457 devmap_devmem_large_page_setup(dhp); in devmap_devmem_setup()
3458 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_setup()
3459 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_devmem_setup()
3463 bcopy(callbackops, &dhp->dh_callbackops, in devmap_devmem_setup()
3470 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { in devmap_devmem_setup()
3471 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); in devmap_devmem_setup()
3472 dhp->dh_flags |= DEVMAP_LOCK_INITED; in devmap_devmem_setup()
3475 dhp->dh_flags |= DEVMAP_SETUP_DONE; in devmap_devmem_setup()
3485 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_devmem_remap() local
3495 (void *)dhp, roff, rnumber, (uint_t)len); in devmap_devmem_remap()
3497 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); in devmap_devmem_remap()
3503 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || in devmap_devmem_remap()
3504 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) in devmap_devmem_remap()
3511 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_devmem_remap()
3520 HOLD_DHP_LOCK(dhp); in devmap_devmem_remap()
3527 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, in devmap_devmem_remap()
3528 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); in devmap_devmem_remap()
3531 dhp->dh_flags |= DEVMAP_MAPPING_INVALID; in devmap_devmem_remap()
3532 dhp->dh_pfn = PFN_INVALID; in devmap_devmem_remap()
3535 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID; in devmap_devmem_remap()
3548 mr.map_prot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_remap()
3562 RELE_DHP_LOCK(dhp); in devmap_devmem_remap()
3569 dhp->dh_pfn = pfn; in devmap_devmem_remap()
3570 dhp->dh_hat_attr = hat_flags; in devmap_devmem_remap()
3574 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; in devmap_devmem_remap()
3576 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; in devmap_devmem_remap()
3577 dhp->dh_roff = ptob(btop(roff)); in devmap_devmem_remap()
3580 devmap_devmem_large_page_setup(dhp); in devmap_devmem_remap()
3581 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_remap()
3582 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_devmem_remap()
3584 RELE_DHP_LOCK(dhp); in devmap_devmem_remap()
3599 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_umem_setup() local
3608 (void *)dhp, off, cookie, len); in devmap_umem_setup()
3610 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); in devmap_umem_setup()
3626 if (dhp->dh_flags & DEVMAP_SETUP_DONE) in devmap_umem_setup()
3629 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_umem_setup()
3641 dhp->dh_cookie = cookie; in devmap_umem_setup()
3642 dhp->dh_roff = ptob(btop(off)); in devmap_umem_setup()
3643 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; in devmap_umem_setup()
3645 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); in devmap_umem_setup()
3647 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); in devmap_umem_setup()
3671 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); in devmap_umem_setup()
3673 dhp->dh_len = ptob(btopr(len)); in devmap_umem_setup()
3674 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_umem_setup()
3675 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_umem_setup()
3678 bcopy(callbackops, &dhp->dh_callbackops, in devmap_umem_setup()
3684 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { in devmap_umem_setup()
3685 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); in devmap_umem_setup()
3686 dhp->dh_flags |= DEVMAP_LOCK_INITED; in devmap_umem_setup()
3689 dhp->dh_flags |= DEVMAP_SETUP_DONE; in devmap_umem_setup()
3699 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_umem_remap() local
3704 (void *)dhp, off, cookie, len); in devmap_umem_remap()
3706 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); in devmap_umem_remap()
3716 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || in devmap_umem_remap()
3717 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) in devmap_umem_remap()
3728 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_umem_remap()
3735 HOLD_DHP_LOCK(dhp); in devmap_umem_remap()
3741 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, in devmap_umem_remap()
3742 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); in devmap_umem_remap()
3744 dhp->dh_cookie = cookie; in devmap_umem_remap()
3745 dhp->dh_roff = ptob(btop(off)); in devmap_umem_remap()
3746 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; in devmap_umem_remap()
3748 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); in devmap_umem_remap()
3750 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); in devmap_umem_remap()
3753 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; in devmap_umem_remap()
3755 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_umem_remap()
3756 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_umem_remap()
3757 RELE_DHP_LOCK(dhp); in devmap_umem_remap()
3768 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_set_ctx_timeout() local
3772 (void *)dhp, ticks); in devmap_set_ctx_timeout()
3773 dhp->dh_timeout_length = ticks; in devmap_set_ctx_timeout()
3777 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off, in devmap_default_access() argument
3786 return (devmap_load(dhp, off, len, type, rw)); in devmap_default_access()