90 #define	SFMMU_VALIDATE_HMERID(hat, rid, saddr, len)			\  argument
96 		ASSERT(SF_RGNMAP_TEST(hat->sfmmu_hmeregion_map, rid));	\
97 		ASSERT((hat) != ksfmmup);				\
98 		_srdp = (hat)->sfmmu_srdp;				\
145 #define	SFMMU_VALIDATE_HMERID(hat, rid, addr, len)  argument
397 static caddr_t	sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t,
399 static caddr_t	sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t,
410 static void	hat_do_memload_array(struct hat *, caddr_t, size_t,
412 static void	hat_do_memload(struct hat *, caddr_t, struct page *,
414 static void	sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **,
416 void		sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *,
433 static void	sfmmu_vac_conflict(struct hat *, caddr_t, page_t *);
442 static void	sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *);
443 static void	sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int);
523 static void	iment_add(struct ism_ment *,  struct hat *);
524 static void	iment_sub(struct ism_ment *, struct hat *);
1458 struct hat *
1868 hat_setup(struct hat *sfmmup, int allocflag)  in hat_setup()
1918 hat_free_start(struct hat *sfmmup)  in hat_free_start()
1933 hat_free_end(struct hat *sfmmup)  in hat_free_end()
1984 hat_swapin(struct hat *hat)  in hat_swapin()  argument
1986 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_swapin()
1995 hat_swapout(struct hat *sfmmup)  in hat_swapout()
2130 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len,  in hat_dup()  argument
2138 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_dup()
2141 	ASSERT(hat != ksfmmup);  in hat_dup()
2143 	ASSERT(flag != HAT_DUP_ALL || hat->sfmmu_srdp == newhat->sfmmu_srdp);  in hat_dup()
2149 	if (flag == HAT_DUP_SRD && ((srdp = hat->sfmmu_srdp) != NULL)) {  in hat_dup()
2162 		newhat->sfmmu_rtteflags = hat->sfmmu_rtteflags;  in hat_dup()
2163 		if (hat->sfmmu_flags & HAT_4MTEXT_FLAG) {  in hat_dup()
2168 		if ((scdp = hat->sfmmu_scdp) != NULL &&  in hat_dup()
2179 				    hat->sfmmu_ismttecnt[i];  in hat_dup()
2181 				    hat->sfmmu_scdismttecnt[i];  in hat_dup()
2190 		hat->sfmmu_clrbin = get_color_start(hat->sfmmu_as);  in hat_dup()
2196 hat_memload(struct hat *hat, caddr_t addr, struct page *pp,  in hat_memload()  argument
2199 	hat_do_memload(hat, addr, pp, attr, flags,  in hat_memload()
2204 hat_memload_region(struct hat *hat, caddr_t addr, struct page *pp,  in hat_memload_region()  argument
2209 	    hat->sfmmu_xhat_provider != NULL) {  in hat_memload_region()
2210 		hat_do_memload(hat, addr, pp, attr, flags,  in hat_memload_region()
2216 	hat_do_memload(hat, addr, pp, attr, flags, rid);  in hat_memload_region()
2225 hat_do_memload(struct hat *hat, caddr_t addr, struct page *pp,  in hat_do_memload()  argument
2231 	ASSERT(hat != NULL);  in hat_do_memload()
2236 	SFMMU_VALIDATE_HMERID(hat, rid, addr, MMU_PAGESIZE);  in hat_do_memload()
2243 	if (hat->sfmmu_xhat_provider) {  in hat_do_memload()
2246 		XHAT_MEMLOAD(hat, addr, pp, attr, flags);  in hat_do_memload()
2250 	ASSERT((hat == ksfmmup) || AS_LOCK_HELD(hat->sfmmu_as));  in hat_do_memload()
2256 	if (hat->sfmmu_rmstat)  in hat_do_memload()
2257 		hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr);  in hat_do_memload()
2260 	if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) &&  in hat_do_memload()
2270 	(void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags, rid);  in hat_do_memload()
2276 		sfmmu_check_page_sizes(hat, 1);  in hat_do_memload()
2289 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn,  in hat_devload()  argument
2296 	ASSERT(hat != NULL);  in hat_devload()
2298 	if (hat->sfmmu_xhat_provider) {  in hat_devload()
2299 		XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags);  in hat_devload()
2305 	ASSERT((hat == ksfmmup) || AS_LOCK_HELD(hat->sfmmu_as));  in hat_devload()
2313 	if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) &&  in hat_devload()
2344 	if (hat->sfmmu_rmstat)  in hat_devload()
2345 		hat_resvstat(len, hat->sfmmu_as, addr);  in hat_devload()
2375 			(void) sfmmu_tteload_array(hat, &tte, addr, &pp,  in hat_devload()
2391 			(void) sfmmu_tteload_array(hat, &tte, addr, &pp,  in hat_devload()
2401 			(void) sfmmu_tteload_array(hat, &tte, addr, &pp,  in hat_devload()
2411 			(void) sfmmu_tteload_array(hat, &tte, addr, &pp,  in hat_devload()
2418 			(void) sfmmu_tteload_array(hat, &tte, addr, &pp,  in hat_devload()
2430 		sfmmu_check_page_sizes(hat, 1);  in hat_devload()
2435 hat_memload_array(struct hat *hat, caddr_t addr, size_t len,  in hat_memload_array()  argument
2438 	hat_do_memload_array(hat, addr, len, pps, attr, flags,  in hat_memload_array()
2443 hat_memload_array_region(struct hat *hat, caddr_t addr, size_t len,  in hat_memload_array_region()  argument
2449 	    hat->sfmmu_xhat_provider != NULL) {  in hat_memload_array_region()
2450 		hat_do_memload_array(hat, addr, len, pps, attr, flags,  in hat_memload_array_region()
2456 	hat_do_memload_array(hat, addr, len, pps, attr, flags, rid);  in hat_memload_array_region()
2471 hat_do_memload_array(struct hat *hat, caddr_t addr, size_t len,  in hat_do_memload_array()  argument
2482 	SFMMU_VALIDATE_HMERID(hat, rid, addr, len);  in hat_do_memload_array()
2484 	if (hat->sfmmu_xhat_provider) {  in hat_do_memload_array()
2486 		XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags);  in hat_do_memload_array()
2490 	if (hat->sfmmu_rmstat)  in hat_do_memload_array()
2491 		hat_resvstat(len, hat->sfmmu_as, addr);  in hat_do_memload_array()
2494 	if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) &&  in hat_do_memload_array()
2513 		sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs,  in hat_do_memload_array()
2546 				if (!sfmmu_tteload_array(hat, &tte, addr,  in hat_do_memload_array()
2562 			sfmmu_memload_batchsmall(hat, addr, pps, attr, flags,  in hat_do_memload_array()
2571 		sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs,  in hat_do_memload_array()
2579 		sfmmu_check_page_sizes(hat, 1);  in hat_do_memload_array()
2587 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps,  in sfmmu_memload_batchsmall()  argument
2600 		hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K,  in sfmmu_memload_batchsmall()
2607 		hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr,  in sfmmu_memload_batchsmall()
2621 			(void) sfmmu_tteload_addentry(hat, hmeblkp, &tte,  in sfmmu_memload_batchsmall()
2703 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp,  in sfmmu_tteload()
3963 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len)  in hat_unlock()
4034 hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len,  in hat_unlock_region()
4762 hat_probe(struct hat *sfmmup, caddr_t addr)  in hat_probe()
4788 hat_getpagesize(struct hat *sfmmup, caddr_t addr)  in hat_getpagesize()
4809 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr)  in hat_getattr()
4836 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr)  in hat_setattr()  argument
4838 	if (hat->sfmmu_xhat_provider) {  in hat_setattr()
4839 		XHAT_SETATTR(hat, addr, len, attr);  in hat_setattr()
4847 		ASSERT(hat->sfmmu_as != NULL);  in hat_setattr()
4848 		if (hat->sfmmu_as->a_xhat != NULL)  in hat_setattr()
4849 			xhat_setattr_all(hat->sfmmu_as, addr, len, attr);  in hat_setattr()
4852 	sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR);  in hat_setattr()
4860 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr)  in hat_chgattr()  argument
4862 	if (hat->sfmmu_xhat_provider) {  in hat_chgattr()
4863 		XHAT_CHGATTR(hat, addr, len, attr);  in hat_chgattr()
4871 		ASSERT(hat->sfmmu_as != NULL);  in hat_chgattr()
4872 		if (hat->sfmmu_as->a_xhat != NULL)  in hat_chgattr()
4873 			xhat_chgattr_all(hat->sfmmu_as, addr, len, attr);  in hat_chgattr()
4876 	sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR);  in hat_chgattr()
4883 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr)  in hat_clrattr()  argument
4885 	if (hat->sfmmu_xhat_provider) {  in hat_clrattr()
4886 		XHAT_CLRATTR(hat, addr, len, attr);  in hat_clrattr()
4894 		ASSERT(hat->sfmmu_as != NULL);  in hat_clrattr()
4895 		if (hat->sfmmu_as->a_xhat != NULL)  in hat_clrattr()
4896 			xhat_clrattr_all(hat->sfmmu_as, addr, len, attr);  in hat_clrattr()
4899 	sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR);  in hat_clrattr()
4906 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr,  in sfmmu_chgattr()
5002 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,  in sfmmu_hblk_chgattr()
5236 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot)  in hat_chgprot()
5534 	struct hat		*sfmmup,  in hat_unload_large_virtual()
5677 	struct hat *sfmmup,  in hat_unload_callback()
5984 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags)  in hat_unload()
6020 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,  in sfmmu_hblk_unload()
6297 hat_flush_range(struct hat *sfmmup, caddr_t va, size_t size)  in hat_flush_range()
6320 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag)  in hat_sync()
6405 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,  in sfmmu_hblk_sync()
6478 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp)  in sfmmu_ttesync()
7966 hat_getpfnum(struct hat *hat, caddr_t addr)  in hat_getpfnum()  argument
7982 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_getpfnum()
7984 	if (hat == ksfmmup) {  in hat_getpfnum()
8002 		return (sfmmu_uvatopfn(addr, hat, NULL));  in hat_getpfnum()
8010 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup, tte_t *ttep)  in sfmmu_uvatopfn()
8164 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags)  in hat_map()  argument
8166 	ASSERT(hat != NULL);  in hat_map()
8167 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_map()
8505 hat_get_mapped_size(struct hat *hat)  in hat_get_mapped_size()  argument
8510 	if (hat == NULL)  in hat_get_mapped_size()
8513 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_get_mapped_size()
8516 		assize += ((pgcnt_t)hat->sfmmu_ttecnt[i] +  in hat_get_mapped_size()
8517 		    (pgcnt_t)hat->sfmmu_scdrttecnt[i]) * TTEBYTES(i);  in hat_get_mapped_size()
8519 	if (hat->sfmmu_iblk == NULL)  in hat_get_mapped_size()
8523 		assize += ((pgcnt_t)hat->sfmmu_ismttecnt[i] +  in hat_get_mapped_size()
8524 		    (pgcnt_t)hat->sfmmu_scdismttecnt[i]) * TTEBYTES(i);  in hat_get_mapped_size()
8530 hat_stats_enable(struct hat *hat)  in hat_stats_enable()  argument
8534 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_stats_enable()
8536 	hatlockp = sfmmu_hat_enter(hat);  in hat_stats_enable()
8537 	hat->sfmmu_rmstat++;  in hat_stats_enable()
8543 hat_stats_disable(struct hat *hat)  in hat_stats_disable()  argument
8547 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_stats_disable()
8549 	hatlockp = sfmmu_hat_enter(hat);  in hat_stats_disable()
8550 	hat->sfmmu_rmstat--;  in hat_stats_disable()
8560 iment_add(struct ism_ment *iment,  struct hat *ism_hat)  in iment_add()
8573 iment_sub(struct ism_ment *iment, struct hat *ism_hat)  in iment_sub()
8614 hat_share(struct hat *sfmmup, caddr_t addr,  in hat_share()
8615 	struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc)  in hat_share()
8844 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc)  in hat_unshare()
8849 	struct hat	*ism_hatid;  in hat_unshare()
9252 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp)  in sfmmu_vac_conflict()  argument
9254 	struct hat *tmphat;  in sfmmu_vac_conflict()
9334 		if (hmeblkp->hblk_shared || tmphat == hat ||  in sfmmu_vac_conflict()
12210 	struct hat	*ism_hatid;  in find_ism_rid()
13402 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp,  in hat_softlock()  argument
13405 	ASSERT(hat->sfmmu_xhat_provider == NULL);  in hat_softlock()
13485 hat_enter(struct hat *hat)  in hat_enter()  argument
13489 	if (hat != ksfmmup) {  in hat_enter()
13490 		hatlockp = TSB_HASH(hat);  in hat_enter()
13496 hat_exit(struct hat *hat)  in hat_exit()  argument
13500 	if (hat != ksfmmup) {  in hat_exit()
13501 		hatlockp = TSB_HASH(hat);  in hat_exit()
13788 hat_join_srd(struct hat *sfmmup, vnode_t *evp)  in hat_join_srd()
13995 hat_join_region(struct hat *sfmmup,  in hat_join_region()
14290 hat_leave_region(struct hat *sfmmup, hat_region_cookie_t rcookie, uint_t flags)  in hat_leave_region()
14498 hat_dup_region(struct hat *sfmmup, hat_region_cookie_t rcookie)  in hat_dup_region()