86 static void	segkp_insert(struct seg *seg, struct segkp_data *kpd);
87 static void	segkp_delete(struct seg *seg, struct segkp_data *kpd);
91 			struct segkp_data *kpd, size_t len);
93 			size_t len, struct segkp_data *kpd, uint_t flags);
95 			size_t len, struct segkp_data *kpd, uint_t flags);
267 	struct segkp_data *kpd;  in segkp_cache_free()  local
278 		kpd = segkp_cache[i].kpf_list;  in segkp_cache_free()
284 		while (kpd != NULL) {  in segkp_cache_free()
287 			next = kpd->kp_next;  in segkp_cache_free()
288 			segkp_release_internal(seg, kpd, kpd->kp_len);  in segkp_cache_free()
289 			kpd = next;  in segkp_cache_free()
304 	struct segkp_data *kpd = NULL;  in segkp_get()  local
306 	if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {  in segkp_get()
307 		kpd->kp_cookie = -1;  in segkp_get()
308 		return (stom(kpd->kp_base, flags));  in segkp_get()
320 	struct segkp_data *kpd = NULL;  in segkp_cache_get()  local
334 		kpd = freelist->kpf_list;  in segkp_cache_get()
335 		freelist->kpf_list = kpd->kp_next;  in segkp_cache_get()
338 		kpd->kp_next = NULL;  in segkp_cache_get()
339 		segkp_insert(seg, kpd);  in segkp_cache_get()
340 		return (stom(kpd->kp_base, flags));  in segkp_cache_get()
344 	if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {  in segkp_cache_get()
345 		kpd->kp_cookie = index;  in segkp_cache_get()
346 		return (stom(kpd->kp_base, flags));  in segkp_cache_get()
358 	struct segkp_data *kpd = NULL;  in segkp_get_withanonmap()  local
362 	if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) {  in segkp_get_withanonmap()
363 		kpd->kp_cookie = -1;  in segkp_get_withanonmap()
364 		return (stom(kpd->kp_base, flags));  in segkp_get_withanonmap()
391 	struct segkp_data	*kpd;  in segkp_get_internal()  local
414 	if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL)  in segkp_get_internal()
425 		kmem_free(kpd, sizeof (struct segkp_data));  in segkp_get_internal()
434 			kmem_free(kpd, sizeof (struct segkp_data));  in segkp_get_internal()
455 		kpd->kp_anon_idx = anon_idx;  in segkp_get_internal()
456 		kpd->kp_anon = amp->ahp;  in segkp_get_internal()
459 		    kpd, vbase, len, flags, 1);  in segkp_get_internal()
469 			kmem_free(kpd, sizeof (struct segkp_data));  in segkp_get_internal()
475 		kpd->kp_anon_idx = anon_idx;  in segkp_get_internal()
476 		kpd->kp_anon = kpsd->kpsd_anon;  in segkp_get_internal()
479 		    kpd, vbase, len, flags, 1);  in segkp_get_internal()
481 		kpd->kp_anon = NULL;  in segkp_get_internal()
482 		kpd->kp_anon_idx = 0;  in segkp_get_internal()
512 		if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i)  in segkp_get_internal()
515 		if (kpd->kp_anon != NULL) {  in segkp_get_internal()
518 			ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i)  in segkp_get_internal()
526 			(void) anon_set_ptr(kpd->kp_anon, anon_idx + i,  in segkp_get_internal()
559 				kpd->kp_flags = flags;  in segkp_get_internal()
560 				kpd->kp_base = vbase;  in segkp_get_internal()
561 				kpd->kp_len = len;  in segkp_get_internal()
562 				segkp_release_internal(seg, kpd, va - vbase);  in segkp_get_internal()
591 	kpd->kp_flags = flags;  in segkp_get_internal()
592 	kpd->kp_base = vbase;  in segkp_get_internal()
593 	kpd->kp_len = len;  in segkp_get_internal()
594 	segkp_insert(seg, kpd);  in segkp_get_internal()
595 	*tkpd = kpd;  in segkp_get_internal()
596 	return (stom(kpd->kp_base, flags));  in segkp_get_internal()
608 	struct segkp_data *kpd = NULL;  in segkp_release()  local
610 	if ((kpd = segkp_find(seg, vaddr)) == NULL) {  in segkp_release()
615 	if (kpd->kp_cookie != -1) {  in segkp_release()
616 		freelist = &segkp_cache[kpd->kp_cookie];  in segkp_release()
619 			segkp_delete(seg, kpd);  in segkp_release()
620 			kpd->kp_next = freelist->kpf_list;  in segkp_release()
621 			freelist->kpf_list = kpd;  in segkp_release()
627 			kpd->kp_cookie = -1;  in segkp_release()
630 	segkp_release_internal(seg, kpd, kpd->kp_len);  in segkp_release()
639 segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len)  in segkp_release_internal()  argument
652 	ASSERT(kpd != NULL);  in segkp_release_internal()
653 	ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1);  in segkp_release_internal()
657 	if (kpd->kp_cookie == -1) {  in segkp_release_internal()
659 		segkp_delete(seg, kpd);  in segkp_release_internal()
667 	if (kpd->kp_flags & KPD_HASREDZONE)  in segkp_release_internal()
668 		redzone = KPD_REDZONE(kpd);  in segkp_release_internal()
671 	va = kpd->kp_base;  in segkp_release_internal()
674 	    ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));  in segkp_release_internal()
692 		if (kpd->kp_anon) {  in segkp_release_internal()
700 			if (kpd->kp_flags & KPD_LOCKED) {  in segkp_release_internal()
701 				ap = anon_get_ptr(kpd->kp_anon,  in segkp_release_internal()
702 				    kpd->kp_anon_idx + i);  in segkp_release_internal()
716 			if ((kpd->kp_flags & KPD_HASAMP) == 0) {  in segkp_release_internal()
717 				anon_free(kpd->kp_anon, kpd->kp_anon_idx + i,  in segkp_release_internal()
724 			    kpd, va, PAGESIZE, 0, 0);  in segkp_release_internal()
726 			if (kpd->kp_flags & KPD_LOCKED) {  in segkp_release_internal()
749 	if (kpd->kp_flags & KPD_LOCKED) {  in segkp_release_internal()
750 		pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));  in segkp_release_internal()
751 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)  in segkp_release_internal()
756 	vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);  in segkp_release_internal()
757 	kmem_free(kpd, sizeof (struct segkp_data));  in segkp_release_internal()
939 	struct segkp_data	*kpd = NULL;  in segkp_fault()  local
952 	if ((kpd = segkp_find(seg, vaddr)) == NULL)  in segkp_fault()
955 	mutex_enter(&kpd->kp_lock);  in segkp_fault()
958 		ASSERT(!(kpd->kp_flags & KPD_LOCKED));  in segkp_fault()
965 		if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||  in segkp_fault()
966 		    len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {  in segkp_fault()
967 			mutex_exit(&kpd->kp_lock);  in segkp_fault()
971 		if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) {  in segkp_fault()
972 			mutex_exit(&kpd->kp_lock);  in segkp_fault()
975 		kpd->kp_flags |= KPD_LOCKED;  in segkp_fault()
976 		mutex_exit(&kpd->kp_lock);  in segkp_fault()
981 		ASSERT(!(kpd->kp_flags & KPD_NO_ANON));  in segkp_fault()
988 		if ((kpd->kp_flags & KPD_HASREDZONE) &&  in segkp_fault()
989 		    btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd))  in segkp_fault()
998 		if (kpd->kp_flags & KPD_LOCKED) {  in segkp_fault()
999 			mutex_exit(&kpd->kp_lock);  in segkp_fault()
1003 		err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags);  in segkp_fault()
1004 		mutex_exit(&kpd->kp_lock);  in segkp_fault()
1015 		if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) != KPD_LOCKED) {  in segkp_fault()
1020 		if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||  in segkp_fault()
1021 		    len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {  in segkp_fault()
1027 			flags = kpd->kp_flags | KPD_WRITEDIRTY;  in segkp_fault()
1029 			flags = kpd->kp_flags;  in segkp_fault()
1030 		err = segkp_unlock(hat, seg, vaddr, len, kpd, flags);  in segkp_fault()
1031 		kpd->kp_flags &= ~KPD_LOCKED;  in segkp_fault()
1032 		mutex_exit(&kpd->kp_lock);  in segkp_fault()
1035 	mutex_exit(&kpd->kp_lock);  in segkp_fault()
1050 	struct segkp_data *kpd = NULL;  in segkp_checkprot()  local
1054 	if ((kpd = segkp_find(seg, vaddr)) == NULL)  in segkp_checkprot()
1057 	mutex_enter(&kpd->kp_lock);  in segkp_checkprot()
1058 	mbase = stom(kpd->kp_base, kpd->kp_flags);  in segkp_checkprot()
1059 	mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags);  in segkp_checkprot()
1062 		mutex_exit(&kpd->kp_lock);  in segkp_checkprot()
1065 	mutex_exit(&kpd->kp_lock);  in segkp_checkprot()
1094 	struct segkp_data *kpd,  in segkp_load()  argument
1102 	ASSERT(MUTEX_HELD(&kpd->kp_lock));  in segkp_load()
1109 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)  in segkp_load()
1121 	i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;  in segkp_load()
1133 		ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);  in segkp_load()
1148 			    (va - vaddr), kpd, flags);  in segkp_load()
1179 	struct segkp_data *kpd,  in segkp_unlock()  argument
1194 	ASSERT(MUTEX_HELD(&kpd->kp_lock));  in segkp_unlock()
1202 	i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;  in segkp_unlock()
1211 		ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);  in segkp_unlock()
1264 		if ((kpd->kp_flags & KPD_NO_ANON) == 0)  in segkp_unlock()
1275 segkp_insert(struct seg *seg, struct segkp_data *kpd)  in segkp_insert()  argument
1284 	index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));  in segkp_insert()
1286 	kpd->kp_next = kpsd->kpsd_hash[index];  in segkp_insert()
1287 	kpsd->kpsd_hash[index] = kpd;  in segkp_insert()
1295 segkp_delete(struct seg *seg, struct segkp_data *kpd)  in segkp_delete()  argument
1303 	index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));  in segkp_delete()
1306 		if (*kpp == kpd) {  in segkp_delete()
1307 			*kpp = kpd->kp_next;  in segkp_delete()
1329 	struct segkp_data *kpd;  in segkp_find()  local
1336 		for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;  in segkp_find()
1337 		    kpd = kpd->kp_next) {  in segkp_find()
1338 			if (vaddr >= kpd->kp_base &&  in segkp_find()
1339 			    vaddr < kpd->kp_base + kpd->kp_len) {  in segkp_find()
1341 				return (kpd);  in segkp_find()
1357 	struct segkp_data *kpd;  in swapsize()  local
1359 	if ((kpd = segkp_find(segkp, v)) != NULL)  in swapsize()
1360 		return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));  in swapsize()
1372 	struct segkp_data *kpd;  in segkp_dump()  local
1376 		for (kpd = kpsd->kpsd_hash[i];  in segkp_dump()
1377 		    kpd != NULL; kpd = kpd->kp_next) {  in segkp_dump()
1382 			addr = kpd->kp_base;  in segkp_dump()
1383 			eaddr = addr + kpd->kp_len;  in segkp_dump()