Lines Matching refs:dhp
89 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ argument
90 { mutex_enter(&dhp->dh_lock); }
92 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ argument
93 { mutex_exit(&dhp->dh_lock); }
152 #define dhp_is_devmem(dhp) \ argument
153 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
155 #define dhp_is_pmem(dhp) \ argument
156 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
158 #define dhp_is_kpmem(dhp) \ argument
159 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
246 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
249 static void free_devmap_handle(devmap_handle_t *dhp);
251 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
254 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
256 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
258 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
260 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
263 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
266 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
366 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data; in segdev_create() local
389 sdp->devmap_data = dhp; in segdev_create()
391 sdp->devmap_data = dhp = NULL; in segdev_create()
412 while (dhp != NULL) { in segdev_create()
413 dhp->dh_seg = seg; in segdev_create()
414 dhp = dhp->dh_next; in segdev_create()
465 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; in segdev_dup() local
470 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg); in segdev_dup()
473 (void *)dhp, (void *)seg)); in segdev_dup()
515 if (dhp != NULL) { in segdev_dup()
516 ret = devmap_handle_dup(dhp, in segdev_dup()
521 ret, (void *)dhp, (void *)seg); in segdev_dup()
524 ret, (void *)dhp, (void *)seg)); in segdev_dup()
542 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp, in devmap_handle_dup() argument
549 while (dhp != NULL) { in devmap_handle_dup()
553 HOLD_DHP_LOCK(dhp); in devmap_handle_dup()
554 bcopy(dhp, newdhp, sizeof (devmap_handle_t)); in devmap_handle_dup()
555 RELE_DHP_LOCK(dhp); in devmap_handle_dup()
566 if (dhp->dh_softlock != NULL) in devmap_handle_dup()
570 if (dhp->dh_ctx != NULL) in devmap_handle_dup()
589 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp, in devmap_handle_dup()
605 (void *)dhp)); in devmap_handle_dup()
611 dhp = dhp->dh_next; in devmap_handle_dup()
631 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; in segdev_unmap() local
642 (void *)dhp, (void *)seg, (void *)addr, len); in segdev_unmap()
645 (void *)dhp, (void *)seg, (void *)addr, len)); in segdev_unmap()
671 if (dhp != NULL) { in segdev_unmap()
677 dhpp = tdhp = devmap_find_handle(dhp, addr); in segdev_unmap()
754 if (dhp != NULL) in segdev_unmap()
755 devmap_handle_unmap_head(dhp, len); in segdev_unmap()
786 if (dhp != NULL) in segdev_unmap()
787 devmap_handle_unmap_tail(dhp, addr); in segdev_unmap()
856 if (dhp == NULL) { in segdev_unmap()
860 while (dhp != NULL) { in segdev_unmap()
861 callbackops = &dhp->dh_callbackops; in segdev_unmap()
863 "segdev_unmap: dhp=%p addr=%p", dhp, addr); in segdev_unmap()
865 (void *)dhp, (void *)addr, in segdev_unmap()
866 (void *)dhp->dh_uvaddr, dhp->dh_len)); in segdev_unmap()
868 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
869 dhpp = dhp->dh_next; in segdev_unmap()
870 dhp->dh_next = NULL; in segdev_unmap()
871 dhp = dhpp; in segdev_unmap()
872 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
873 dhp = dhp->dh_next; in segdev_unmap()
874 } else if (addr > dhp->dh_uvaddr && in segdev_unmap()
875 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
882 HOLD_DHP_LOCK(dhp); in segdev_unmap()
883 bcopy(dhp, newdhp, sizeof (devmap_handle_t)); in segdev_unmap()
884 RELE_DHP_LOCK(dhp); in segdev_unmap()
886 newdhp->dh_next = dhp->dh_next; in segdev_unmap()
887 if (dhp->dh_softlock != NULL) in segdev_unmap()
891 if (dhp->dh_ctx != NULL) in segdev_unmap()
899 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in segdev_unmap()
900 off, len, dhp, &dhp->dh_pvtp, in segdev_unmap()
902 mlen = len + (addr - dhp->dh_uvaddr); in segdev_unmap()
906 dhp->dh_len = addr - dhp->dh_uvaddr; in segdev_unmap()
907 dhpp = dhp->dh_next; in segdev_unmap()
908 dhp->dh_next = NULL; in segdev_unmap()
909 dhp = dhpp; in segdev_unmap()
910 } else if ((addr > dhp->dh_uvaddr) && in segdev_unmap()
911 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) { in segdev_unmap()
912 mlen = dhp->dh_len + dhp->dh_uvaddr - addr; in segdev_unmap()
917 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in segdev_unmap()
918 off, mlen, (devmap_cookie_t *)dhp, in segdev_unmap()
919 &dhp->dh_pvtp, NULL, NULL); in segdev_unmap()
921 dhp->dh_len = addr - dhp->dh_uvaddr; in segdev_unmap()
922 dhpp = dhp->dh_next; in segdev_unmap()
923 dhp->dh_next = NULL; in segdev_unmap()
924 dhp = dhpp; in segdev_unmap()
925 nsdp->devmap_data = dhp; in segdev_unmap()
926 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) { in segdev_unmap()
930 dhp->dh_seg = nseg; in segdev_unmap()
931 nsdp->devmap_data = dhp; in segdev_unmap()
932 dhp = devmap_handle_unmap(dhp); in segdev_unmap()
933 nsdp->devmap_data = dhp; /* XX redundant? */ in segdev_unmap()
934 } else if (((addr + len) > dhp->dh_uvaddr) && in segdev_unmap()
935 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) { in segdev_unmap()
936 mlen = addr + len - dhp->dh_uvaddr; in segdev_unmap()
938 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in segdev_unmap()
939 dhp->dh_uoff, mlen, NULL, in segdev_unmap()
940 NULL, dhp, &dhp->dh_pvtp); in segdev_unmap()
941 devmap_handle_reduce_len(dhp, mlen); in segdev_unmap()
942 nsdp->devmap_data = dhp; in segdev_unmap()
943 dhp->dh_seg = nseg; in segdev_unmap()
944 dhp = dhp->dh_next; in segdev_unmap()
946 dhp->dh_seg = nseg; in segdev_unmap()
947 dhp = dhp->dh_next; in segdev_unmap()
960 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len) in devmap_handle_reduce_len() argument
967 ASSERT(len < dhp->dh_len); in devmap_handle_reduce_len()
972 dhp->dh_len -= len; in devmap_handle_reduce_len()
973 dhp->dh_uoff += (offset_t)len; in devmap_handle_reduce_len()
974 dhp->dh_roff += (offset_t)len; in devmap_handle_reduce_len()
975 dhp->dh_uvaddr += len; in devmap_handle_reduce_len()
977 HOLD_DHP_LOCK(dhp); in devmap_handle_reduce_len()
978 cp = dhp->dh_cookie; in devmap_handle_reduce_len()
979 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) { in devmap_handle_reduce_len()
981 dhp->dh_pfn += btop(len); in devmap_handle_reduce_len()
983 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; in devmap_handle_reduce_len()
984 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && in devmap_handle_reduce_len()
985 dhp->dh_roff < ptob(pcp->dp_npages)); in devmap_handle_reduce_len()
987 ASSERT(dhp->dh_roff < cp->size); in devmap_handle_reduce_len()
988 ASSERT(dhp->dh_cvaddr >= cp->cvaddr && in devmap_handle_reduce_len()
989 dhp->dh_cvaddr < (cp->cvaddr + cp->size)); in devmap_handle_reduce_len()
990 ASSERT((dhp->dh_cvaddr + len) <= in devmap_handle_reduce_len()
993 dhp->dh_cvaddr += len; in devmap_handle_reduce_len()
997 RELE_DHP_LOCK(dhp); in devmap_handle_reduce_len()
1005 devmap_handle_unmap(devmap_handle_t *dhp) in devmap_handle_unmap() argument
1007 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops; in devmap_handle_unmap()
1008 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data; in devmap_handle_unmap()
1011 ASSERT(dhp != NULL); in devmap_handle_unmap()
1018 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff, in devmap_handle_unmap()
1019 dhp->dh_len, NULL, NULL, NULL, NULL); in devmap_handle_unmap()
1022 if (dhpp == dhp) { /* releasing first dhp, change sdp data */ in devmap_handle_unmap()
1023 sdp->devmap_data = dhp->dh_next; in devmap_handle_unmap()
1025 while (dhpp->dh_next != dhp) { in devmap_handle_unmap()
1028 dhpp->dh_next = dhp->dh_next; in devmap_handle_unmap()
1030 dhpp = dhp->dh_next; /* return value is next dhp in chain */ in devmap_handle_unmap()
1032 if (dhp->dh_softlock != NULL) in devmap_handle_unmap()
1033 devmap_softlock_rele(dhp); in devmap_handle_unmap()
1035 if (dhp->dh_ctx != NULL) in devmap_handle_unmap()
1036 devmap_ctx_rele(dhp); in devmap_handle_unmap()
1038 if (dhp->dh_flags & DEVMAP_LOCK_INITED) { in devmap_handle_unmap()
1039 mutex_destroy(&dhp->dh_lock); in devmap_handle_unmap()
1041 kmem_free(dhp, sizeof (devmap_handle_t)); in devmap_handle_unmap()
1051 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len) in devmap_handle_unmap_head() argument
1058 while (len >= dhp->dh_len) { in devmap_handle_unmap_head()
1059 len -= dhp->dh_len; in devmap_handle_unmap_head()
1060 dhp = devmap_handle_unmap(dhp); in devmap_handle_unmap_head()
1063 callbackops = &dhp->dh_callbackops; in devmap_handle_unmap_head()
1070 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, in devmap_handle_unmap_head()
1071 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp); in devmap_handle_unmap_head()
1072 devmap_handle_reduce_len(dhp, len); in devmap_handle_unmap_head()
1082 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr) in devmap_handle_unmap_tail() argument
1084 register struct seg *seg = dhp->dh_seg; in devmap_handle_unmap_tail()
1093 maplen = (size_t)(addr - dhp->dh_uvaddr); in devmap_handle_unmap_tail()
1129 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; in segdev_free() local
1132 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg); in segdev_free()
1134 (void *)dhp, (void *)seg)); in segdev_free()
1142 while (dhp != NULL) in segdev_free()
1143 dhp = devmap_handle_unmap(dhp); in segdev_free()
1154 free_devmap_handle(devmap_handle_t *dhp) in free_devmap_handle() argument
1161 while (dhp != NULL) { in free_devmap_handle()
1162 dhpp = dhp->dh_next; in free_devmap_handle()
1163 if (dhp->dh_flags & DEVMAP_LOCK_INITED) { in free_devmap_handle()
1164 mutex_destroy(&dhp->dh_lock); in free_devmap_handle()
1167 if (dhp->dh_softlock != NULL) in free_devmap_handle()
1168 devmap_softlock_rele(dhp); in free_devmap_handle()
1170 if (dhp->dh_ctx != NULL) in free_devmap_handle()
1171 devmap_ctx_rele(dhp); in free_devmap_handle()
1173 kmem_free(dhp, sizeof (devmap_handle_t)); in free_devmap_handle()
1174 dhp = dhpp; in free_devmap_handle()
1343 devmap_handle_t *dhp; in segdev_softunlock() local
1348 dhp = devmap_find_handle(dhp_head, addr); in segdev_softunlock()
1349 ASSERT(dhp != NULL); in segdev_softunlock()
1351 off = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_softunlock()
1353 mlen = MIN(tlen, (dhp->dh_len - off)); in segdev_softunlock()
1358 if (dhp_is_kpmem(dhp)) { in segdev_softunlock()
1360 (struct ddi_umem_cookie *)dhp->dh_cookie, in segdev_softunlock()
1367 if (dhp->dh_callbackops.devmap_access != NULL) { in segdev_softunlock()
1368 devmap_softlock_exit(dhp->dh_softlock, in segdev_softunlock()
1373 dhp = dhp->dh_next; in segdev_softunlock()
1416 devmap_handle_t *dhp) /* devmap handle if any for this page */ in segdev_faultpage() argument
1426 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr); in segdev_faultpage()
1428 (void *)dhp, (void *)seg, (void *)addr)); in segdev_faultpage()
1474 if (dhp == NULL) { in segdev_faultpage()
1486 ASSERT(dhp == devmap_find_handle( in segdev_faultpage()
1489 off = addr - dhp->dh_uvaddr; in segdev_faultpage()
1496 cp = dhp->dh_cookie; in segdev_faultpage()
1497 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { in segdev_faultpage()
1500 pfnum = dhp->dh_pfn + btop(off); in segdev_faultpage()
1502 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; in segdev_faultpage()
1503 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && in segdev_faultpage()
1504 dhp->dh_roff < ptob(pcp->dp_npages)); in segdev_faultpage()
1506 pcp->dp_pparray[btop(off + dhp->dh_roff)]); in segdev_faultpage()
1508 ASSERT(dhp->dh_roff < cp->size); in segdev_faultpage()
1509 ASSERT(dhp->dh_cvaddr >= cp->cvaddr && in segdev_faultpage()
1510 dhp->dh_cvaddr < (cp->cvaddr + cp->size)); in segdev_faultpage()
1511 ASSERT((dhp->dh_cvaddr + off) <= in segdev_faultpage()
1513 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <= in segdev_faultpage()
1519 ASSERT((dhp->dh_roff & in segdev_faultpage()
1523 dhp->dh_roff)]); in segdev_faultpage()
1540 dhp->dh_cvaddr + off); in segdev_faultpage()
1547 prot |= dhp->dh_hat_attr; in segdev_faultpage()
1560 if (pf_is_memory(pfnum) || (dhp != NULL)) { in segdev_faultpage()
1607 devmap_handle_t *dhp; in segdev_fault() local
1629 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) in segdev_fault()
1701 devmap_handle_t *dhpp = dhp; in segdev_fault()
1777 off = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_fault()
1792 mlen = MIN(len, (dhp->dh_len - off)); in segdev_fault()
1794 HOLD_DHP_LOCK(dhp); in segdev_fault()
1800 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) { in segdev_fault()
1801 devmap_get_large_pgsize(dhp, mlen, maddr, in segdev_fault()
1809 if (dhp->dh_callbackops.devmap_access != NULL) { in segdev_fault()
1821 RELE_DHP_LOCK(dhp); in segdev_fault()
1823 err = (*dhp->dh_callbackops.devmap_access)( in segdev_fault()
1824 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw); in segdev_fault()
1831 type, rw, dhp); in segdev_fault()
1833 RELE_DHP_LOCK(dhp); in segdev_fault()
1878 dhp = dhp->dh_next; in segdev_fault()
1881 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr); in segdev_fault()
1909 devmap_handle_t *dhp) /* devmap handle */ in segdev_faultpages() argument
1919 (void *)dhp, (void *)seg, (void *)addr, len); in segdev_faultpages()
1922 (void *)dhp, (void *)seg, (void *)addr, len)); in segdev_faultpages()
1947 if ((dhp != NULL) && dhp_is_kpmem(dhp)) { in segdev_faultpages()
1948 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie; in segdev_faultpages()
1993 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) { in segdev_faultpages()
1997 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { in segdev_faultpages()
2009 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr)); in segdev_faultpages()
2012 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr, in segdev_faultpages()
2027 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) { in segdev_faultpages()
2071 register devmap_handle_t *dhp; in segdev_setprot() local
2095 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) in segdev_setprot()
2101 off = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_setprot()
2103 while (dhp) { in segdev_setprot()
2104 if ((dhp->dh_maxprot & prot) != prot) in segdev_setprot()
2107 if (mlen > (dhp->dh_len - off)) { in segdev_setprot()
2108 mlen -= dhp->dh_len - off; in segdev_setprot()
2109 dhp = dhp->dh_next; in segdev_setprot()
2154 dhp = tdhp = devmap_find_handle(dhp_head, addr); in segdev_setprot()
2166 soff = (ulong_t)(addr - dhp->dh_uvaddr); in segdev_setprot()
2168 mlen = MIN(slen, (dhp->dh_len - soff)); in segdev_setprot()
2169 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr, in segdev_setprot()
2170 dhp->dh_len, HAT_UNLOAD); in segdev_setprot()
2171 dhp = dhp->dh_next; in segdev_setprot()
2499 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr, in devmap_device() argument
2512 (void *)dhp, (void *)addr, off, len); in devmap_device()
2515 (void *)dhp, (void *)addr, off, len)); in devmap_device()
2521 rdhp = maxdhp = dhp; in devmap_device()
2526 maxprot |= dhp->dh_maxprot; in devmap_device()
2528 offset = maxdhp->dh_uoff - dhp->dh_uoff; in devmap_device()
2570 dev_a.dev = dhp->dh_dev; in devmap_device()
2577 dev_a.prot = dhp->dh_prot; in devmap_device()
2583 dev_a.devmap_data = (void *)dhp; in devmap_device()
2595 register devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_do_ctxmgt() local
2606 (void *)dhp, off, len); in devmap_do_ctxmgt()
2608 (void *)dhp, off, len)); in devmap_do_ctxmgt()
2613 devctx = dhp->dh_ctx; in devmap_do_ctxmgt()
2626 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) { in devmap_do_ctxmgt()
2629 devctx, dhp); in devmap_do_ctxmgt()
2642 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw); in devmap_do_ctxmgt()
2654 ret, dhp, devctx); in devmap_do_ctxmgt()
2656 ret, (void *)dhp)); in devmap_do_ctxmgt()
2671 if (dhp->dh_timeout_length > 0) { in devmap_do_ctxmgt()
2675 devctx, dhp->dh_timeout_length); in devmap_do_ctxmgt()
2706 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len, in devmap_roundup() argument
2718 (void *)dhp, offset, len); in devmap_roundup()
2720 (void *)dhp, offset, len)); in devmap_roundup()
2732 base = (ulong_t)ptob(dhp->dh_pfn); in devmap_roundup()
2733 for (level = dhp->dh_mmulevel; level >= 0; level--) { in devmap_roundup()
2736 uvaddr = dhp->dh_uvaddr + (poff - base); in devmap_roundup()
2738 ((poff + pg) <= (base + dhp->dh_len)) && in devmap_roundup()
2745 base, poff, dhp); in devmap_roundup()
2747 base, poff, dhp->dh_pfn)); in devmap_roundup()
2753 *opfn = dhp->dh_pfn + btop(poff - base); in devmap_roundup()
2761 (void *)dhp, level, rlen, pagesize, opfn); in devmap_roundup()
2764 (void *)dhp, level, rlen, *pagesize, *opfn)); in devmap_roundup()
2775 devmap_handle_t *dhp; in devmap_find_handle() local
2780 dhp = dhp_head; in devmap_find_handle()
2781 while (dhp) { in devmap_find_handle()
2782 if (addr >= dhp->dh_uvaddr && in devmap_find_handle()
2783 addr < (dhp->dh_uvaddr + dhp->dh_len)) in devmap_find_handle()
2784 return (dhp); in devmap_find_handle()
2785 dhp = dhp->dh_next; in devmap_find_handle()
2800 register devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_unload() local
2807 (void *)dhp, offset, len); in devmap_unload()
2809 (void *)dhp, offset, len)); in devmap_unload()
2811 soff = (ssize_t)(offset - dhp->dh_uoff); in devmap_unload()
2813 if (soff < 0 || soff >= dhp->dh_len) in devmap_unload()
2829 size = dhp->dh_len - soff; in devmap_unload()
2832 if ((soff + size) > dhp->dh_len) in devmap_unload()
2840 addr = (caddr_t)(soff + dhp->dh_uvaddr); in devmap_unload()
2846 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) { in devmap_unload()
2847 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, in devmap_unload()
2848 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); in devmap_unload()
2850 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size, in devmap_unload()
2861 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr, in devmap_get_large_pgsize() argument
2877 ASSERT(dhp_is_devmem(dhp)); in devmap_get_large_pgsize()
2878 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)); in devmap_get_large_pgsize()
2881 off = (ulong_t)(addr - dhp->dh_uvaddr); in devmap_get_large_pgsize()
2888 len = devmap_roundup(dhp, off, len, &pfn, &pgsize); in devmap_get_large_pgsize()
2891 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn); in devmap_get_large_pgsize()
2896 off = ptob(pfn - dhp->dh_pfn) + pgsize; in devmap_get_large_pgsize()
2946 devmap_softlock_rele(devmap_handle_t *dhp) in devmap_softlock_rele() argument
2948 struct devmap_softlock *slock = dhp->dh_softlock; in devmap_softlock_rele()
2997 devmap_ctx_rele(devmap_handle_t *dhp) in devmap_ctx_rele() argument
2999 struct devmap_ctx *devctx = dhp->dh_ctx; in devmap_ctx_rele()
3068 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_load() local
3069 struct as *asp = dhp->dh_seg->s_as; in devmap_load()
3077 (void *)dhp, offset, len); in devmap_load()
3080 (void *)dhp, offset, len)); in devmap_load()
3090 soff = (ssize_t)(offset - dhp->dh_uoff); in devmap_load()
3092 if (soff < 0 || soff >= dhp->dh_len) in devmap_load()
3108 size = dhp->dh_len - soff; in devmap_load()
3111 if ((soff + size) > dhp->dh_len) in devmap_load()
3119 addr = (caddr_t)(soff + dhp->dh_uvaddr); in devmap_load()
3121 HOLD_DHP_LOCK(dhp); in devmap_load()
3123 dhp->dh_seg, addr, size, type, rw, dhp); in devmap_load()
3124 RELE_DHP_LOCK(dhp); in devmap_load()
3132 register devmap_handle_t *dhp; in devmap_setup() local
3182 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP); in devmap_setup()
3185 dhp_prev->dh_next = dhp; in devmap_setup()
3187 dhp_head = dhp; in devmap_setup()
3188 dhp_prev = dhp; in devmap_setup()
3190 dhp->dh_prot = prot; in devmap_setup()
3191 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot; in devmap_setup()
3192 dhp->dh_dev = dev; in devmap_setup()
3193 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE; in devmap_setup()
3194 dhp->dh_uoff = map_off; in devmap_setup()
3202 if ((ret = cdev_devmap(dev, dhp, map_off, in devmap_setup()
3213 callbackops = &dhp->dh_callbackops; in devmap_setup()
3231 dhp->dh_softlock = devmap_softlock_init(dev, in devmap_setup()
3233 dhp->dh_ctx = devmap_ctxinit(dev, in devmap_setup()
3246 slock = dhp->dh_softlock; in devmap_setup()
3247 if (slock != dhp->dh_softlock) { in devmap_setup()
3275 dhp = dhp_head; in devmap_setup()
3278 while (dhp != NULL) { in devmap_setup()
3279 callbackops = &dhp->dh_callbackops; in devmap_setup()
3280 dhp->dh_uvaddr = addr; in devmap_setup()
3281 dhp_curr = dhp; in devmap_setup()
3283 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp, in devmap_setup()
3285 dhp->dh_len, &dhp->dh_pvtp); in devmap_setup()
3293 dhp = dhp_head; in devmap_setup()
3295 while (dhp != dhp_curr) { in devmap_setup()
3296 callbackops = &dhp->dh_callbackops; in devmap_setup()
3299 dhp, dhp->dh_pvtp, in devmap_setup()
3300 map_off, dhp->dh_len, in devmap_setup()
3303 map_off += dhp->dh_len; in devmap_setup()
3304 dhp = dhp->dh_next; in devmap_setup()
3312 map_off += dhp->dh_len; in devmap_setup()
3313 addr += dhp->dh_len; in devmap_setup()
3314 dhp = dhp->dh_next; in devmap_setup()
3338 devmap_devmem_large_page_setup(devmap_handle_t *dhp) in devmap_devmem_large_page_setup() argument
3340 ASSERT(dhp_is_devmem(dhp)); in devmap_devmem_large_page_setup()
3341 dhp->dh_mmulevel = 0; in devmap_devmem_large_page_setup()
3352 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) { in devmap_devmem_large_page_setup()
3356 base = (ulong_t)ptob(dhp->dh_pfn); in devmap_devmem_large_page_setup()
3359 if ((dhp->dh_len < pgsize) || in devmap_devmem_large_page_setup()
3360 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr, in devmap_devmem_large_page_setup()
3365 dhp->dh_mmulevel = level - 1; in devmap_devmem_large_page_setup()
3367 if (dhp->dh_mmulevel > 0) { in devmap_devmem_large_page_setup()
3368 dhp->dh_flags |= DEVMAP_FLAG_LARGE; in devmap_devmem_large_page_setup()
3370 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; in devmap_devmem_large_page_setup()
3384 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_devmem_setup() local
3392 (void *)dhp, roff, rnumber, (uint_t)len); in devmap_devmem_setup()
3394 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); in devmap_devmem_setup()
3399 if (dhp->dh_flags & DEVMAP_SETUP_DONE) in devmap_devmem_setup()
3402 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_devmem_setup()
3416 dhp->dh_pfn = PFN_INVALID; in devmap_devmem_setup()
3434 mr.map_prot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_setup()
3444 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn); in devmap_devmem_setup()
3445 dhp->dh_hat_attr = hp->ah_hat_flags; in devmap_devmem_setup()
3452 ASSERT(!pf_is_memory(dhp->dh_pfn)); in devmap_devmem_setup()
3455 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); in devmap_devmem_setup()
3456 dhp->dh_len = ptob(btopr(len)); in devmap_devmem_setup()
3458 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; in devmap_devmem_setup()
3459 dhp->dh_roff = ptob(btop(roff)); in devmap_devmem_setup()
3462 devmap_devmem_large_page_setup(dhp); in devmap_devmem_setup()
3463 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_setup()
3464 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_devmem_setup()
3468 bcopy(callbackops, &dhp->dh_callbackops, in devmap_devmem_setup()
3475 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { in devmap_devmem_setup()
3476 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); in devmap_devmem_setup()
3477 dhp->dh_flags |= DEVMAP_LOCK_INITED; in devmap_devmem_setup()
3480 dhp->dh_flags |= DEVMAP_SETUP_DONE; in devmap_devmem_setup()
3490 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_devmem_remap() local
3500 (void *)dhp, roff, rnumber, (uint_t)len); in devmap_devmem_remap()
3502 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); in devmap_devmem_remap()
3508 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || in devmap_devmem_remap()
3509 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) in devmap_devmem_remap()
3516 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_devmem_remap()
3525 HOLD_DHP_LOCK(dhp); in devmap_devmem_remap()
3532 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, in devmap_devmem_remap()
3533 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); in devmap_devmem_remap()
3536 dhp->dh_flags |= DEVMAP_MAPPING_INVALID; in devmap_devmem_remap()
3537 dhp->dh_pfn = PFN_INVALID; in devmap_devmem_remap()
3540 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID; in devmap_devmem_remap()
3553 mr.map_prot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_remap()
3567 RELE_DHP_LOCK(dhp); in devmap_devmem_remap()
3574 dhp->dh_pfn = pfn; in devmap_devmem_remap()
3575 dhp->dh_hat_attr = hat_flags; in devmap_devmem_remap()
3579 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; in devmap_devmem_remap()
3581 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; in devmap_devmem_remap()
3582 dhp->dh_roff = ptob(btop(roff)); in devmap_devmem_remap()
3585 devmap_devmem_large_page_setup(dhp); in devmap_devmem_remap()
3586 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_devmem_remap()
3587 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_devmem_remap()
3589 RELE_DHP_LOCK(dhp); in devmap_devmem_remap()
3604 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_umem_setup() local
3609 (void *)dhp, off, cookie, len); in devmap_umem_setup()
3611 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); in devmap_umem_setup()
3627 if (dhp->dh_flags & DEVMAP_SETUP_DONE) in devmap_umem_setup()
3630 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_umem_setup()
3642 dhp->dh_cookie = cookie; in devmap_umem_setup()
3643 dhp->dh_roff = ptob(btop(off)); in devmap_umem_setup()
3644 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; in devmap_umem_setup()
3646 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); in devmap_umem_setup()
3648 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); in devmap_umem_setup()
3672 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); in devmap_umem_setup()
3674 dhp->dh_len = ptob(btopr(len)); in devmap_umem_setup()
3675 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_umem_setup()
3676 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_umem_setup()
3679 bcopy(callbackops, &dhp->dh_callbackops, in devmap_umem_setup()
3685 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { in devmap_umem_setup()
3686 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); in devmap_umem_setup()
3687 dhp->dh_flags |= DEVMAP_LOCK_INITED; in devmap_umem_setup()
3690 dhp->dh_flags |= DEVMAP_SETUP_DONE; in devmap_umem_setup()
3700 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_umem_remap() local
3705 (void *)dhp, off, cookie, len); in devmap_umem_remap()
3707 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); in devmap_umem_remap()
3713 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || in devmap_umem_remap()
3714 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) in devmap_umem_remap()
3725 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) in devmap_umem_remap()
3732 HOLD_DHP_LOCK(dhp); in devmap_umem_remap()
3738 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, in devmap_umem_remap()
3739 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); in devmap_umem_remap()
3741 dhp->dh_cookie = cookie; in devmap_umem_remap()
3742 dhp->dh_roff = ptob(btop(off)); in devmap_umem_remap()
3743 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; in devmap_umem_remap()
3745 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); in devmap_umem_remap()
3747 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); in devmap_umem_remap()
3750 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; in devmap_umem_remap()
3752 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; in devmap_umem_remap()
3753 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); in devmap_umem_remap()
3754 RELE_DHP_LOCK(dhp); in devmap_umem_remap()
3765 devmap_handle_t *dhp = (devmap_handle_t *)dhc; in devmap_set_ctx_timeout() local
3769 (void *)dhp, ticks); in devmap_set_ctx_timeout()
3770 dhp->dh_timeout_length = ticks; in devmap_set_ctx_timeout()
3774 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off, in devmap_default_access() argument
3783 return (devmap_load(dhp, off, len, type, rw)); in devmap_default_access()