900 rms_int_pcpu(struct rmslock *rms)  in rms_int_pcpu()  argument
904 	return (zpcpu_get(rms->pcpu));  in rms_int_pcpu()
908 rms_int_remote_pcpu(struct rmslock *rms, int cpu)  in rms_int_remote_pcpu()  argument
911 	return (zpcpu_get_cpu(rms->pcpu, cpu));  in rms_int_remote_pcpu()
915 rms_int_influx_enter(struct rmslock *rms, struct rmslock_pcpu *pcpu)  in rms_int_influx_enter()  argument
924 rms_int_influx_exit(struct rmslock *rms, struct rmslock_pcpu *pcpu)  in rms_int_influx_exit()  argument
934 rms_int_debug_readers_inc(struct rmslock *rms)  in rms_int_debug_readers_inc()  argument
937 	old = atomic_fetchadd_int(&rms->debug_readers, 1);  in rms_int_debug_readers_inc()
942 rms_int_debug_readers_dec(struct rmslock *rms)  in rms_int_debug_readers_dec()  argument
946 	old = atomic_fetchadd_int(&rms->debug_readers, -1);  in rms_int_debug_readers_dec()
951 rms_int_debug_readers_inc(struct rmslock *rms)  in rms_int_debug_readers_inc()  argument
956 rms_int_debug_readers_dec(struct rmslock *rms)  in rms_int_debug_readers_dec()  argument
962 rms_int_readers_inc(struct rmslock *rms, struct rmslock_pcpu *pcpu)  in rms_int_readers_inc()  argument
966 	rms_int_debug_readers_inc(rms);  in rms_int_readers_inc()
971 rms_int_readers_dec(struct rmslock *rms, struct rmslock_pcpu *pcpu)  in rms_int_readers_dec()  argument
975 	rms_int_debug_readers_dec(rms);  in rms_int_readers_dec()
983 rms_init(struct rmslock *rms, const char *name)  in rms_init()  argument
986 	rms->owner = RMS_NOOWNER;  in rms_init()
987 	rms->writers = 0;  in rms_init()
988 	rms->readers = 0;  in rms_init()
989 	rms->debug_readers = 0;  in rms_init()
990 	mtx_init(&rms->mtx, name, NULL, MTX_DEF | MTX_NEW);  in rms_init()
991 	rms->pcpu = uma_zalloc_pcpu(pcpu_zone_8, M_WAITOK | M_ZERO);  in rms_init()
995 rms_destroy(struct rmslock *rms)  in rms_destroy()  argument
998 	MPASS(rms->writers == 0);  in rms_destroy()
999 	MPASS(rms->readers == 0);  in rms_destroy()
1000 	mtx_destroy(&rms->mtx);  in rms_destroy()
1001 	uma_zfree_pcpu(pcpu_zone_8, rms->pcpu);  in rms_destroy()
1005 rms_rlock_fallback(struct rmslock *rms)  in rms_rlock_fallback()  argument
1008 	rms_int_influx_exit(rms, rms_int_pcpu(rms));  in rms_rlock_fallback()
1011 	mtx_lock(&rms->mtx);  in rms_rlock_fallback()
1012 	while (rms->writers > 0)  in rms_rlock_fallback()
1013 		msleep(&rms->readers, &rms->mtx, PUSER - 1, mtx_name(&rms->mtx), 0);  in rms_rlock_fallback()
1015 	rms_int_readers_inc(rms, rms_int_pcpu(rms));  in rms_rlock_fallback()
1016 	mtx_unlock(&rms->mtx);  in rms_rlock_fallback()
1022 rms_rlock(struct rmslock *rms)  in rms_rlock()  argument
1026 	rms_assert_rlock_ok(rms);  in rms_rlock()
1027 	MPASS(atomic_load_ptr(&rms->owner) != curthread);  in rms_rlock()
1030 	pcpu = rms_int_pcpu(rms);  in rms_rlock()
1031 	rms_int_influx_enter(rms, pcpu);  in rms_rlock()
1033 	if (__predict_false(rms->writers > 0)) {  in rms_rlock()
1034 		rms_rlock_fallback(rms);  in rms_rlock()
1038 	rms_int_readers_inc(rms, pcpu);  in rms_rlock()
1040 	rms_int_influx_exit(rms, pcpu);  in rms_rlock()
1046 rms_try_rlock(struct rmslock *rms)  in rms_try_rlock()  argument
1050 	MPASS(atomic_load_ptr(&rms->owner) != curthread);  in rms_try_rlock()
1053 	pcpu = rms_int_pcpu(rms);  in rms_try_rlock()
1054 	rms_int_influx_enter(rms, pcpu);  in rms_try_rlock()
1056 	if (__predict_false(rms->writers > 0)) {  in rms_try_rlock()
1057 		rms_int_influx_exit(rms, pcpu);  in rms_try_rlock()
1062 	rms_int_readers_inc(rms, pcpu);  in rms_try_rlock()
1064 	rms_int_influx_exit(rms, pcpu);  in rms_try_rlock()
1071 rms_runlock_fallback(struct rmslock *rms)  in rms_runlock_fallback()  argument
1074 	rms_int_influx_exit(rms, rms_int_pcpu(rms));  in rms_runlock_fallback()
1077 	mtx_lock(&rms->mtx);  in rms_runlock_fallback()
1078 	MPASS(rms->writers > 0);  in rms_runlock_fallback()
1079 	MPASS(rms->readers > 0);  in rms_runlock_fallback()
1080 	MPASS(rms->debug_readers == rms->readers);  in rms_runlock_fallback()
1081 	rms_int_debug_readers_dec(rms);  in rms_runlock_fallback()
1082 	rms->readers--;  in rms_runlock_fallback()
1083 	if (rms->readers == 0)  in rms_runlock_fallback()
1084 		wakeup_one(&rms->writers);  in rms_runlock_fallback()
1085 	mtx_unlock(&rms->mtx);  in rms_runlock_fallback()
1090 rms_runlock(struct rmslock *rms)  in rms_runlock()  argument
1095 	pcpu = rms_int_pcpu(rms);  in rms_runlock()
1096 	rms_int_influx_enter(rms, pcpu);  in rms_runlock()
1098 	if (__predict_false(rms->writers > 0)) {  in rms_runlock()
1099 		rms_runlock_fallback(rms);  in rms_runlock()
1103 	rms_int_readers_dec(rms, pcpu);  in rms_runlock()
1105 	rms_int_influx_exit(rms, pcpu);  in rms_runlock()
1111 	struct rmslock *rms;  member
1120 	struct rmslock *rms;  in rms_action_func()  local
1123 	rms = rmsipi->rms;  in rms_action_func()
1124 	pcpu = rms_int_pcpu(rms);  in rms_action_func()
1129 		atomic_add_int(&rms->readers, pcpu->readers);  in rms_action_func()
1140 	struct rmslock *rms;  in rms_wait_func()  local
1143 	rms = rmsipi->rms;  in rms_wait_func()
1144 	pcpu = rms_int_remote_pcpu(rms, cpu);  in rms_wait_func()
1152 rms_assert_no_pcpu_readers(struct rmslock *rms)  in rms_assert_no_pcpu_readers()  argument
1158 		pcpu = rms_int_remote_pcpu(rms, cpu);  in rms_assert_no_pcpu_readers()
1167 rms_assert_no_pcpu_readers(struct rmslock *rms)  in rms_assert_no_pcpu_readers()  argument
1173 rms_wlock_switch(struct rmslock *rms)  in rms_wlock_switch()  argument
1177 	MPASS(rms->readers == 0);  in rms_wlock_switch()
1178 	MPASS(rms->writers == 1);  in rms_wlock_switch()
1180 	rmsipi.rms = rms;  in rms_wlock_switch()
1191 rms_wlock(struct rmslock *rms)  in rms_wlock()  argument
1195 	MPASS(atomic_load_ptr(&rms->owner) != curthread);  in rms_wlock()
1197 	mtx_lock(&rms->mtx);  in rms_wlock()
1198 	rms->writers++;  in rms_wlock()
1199 	if (rms->writers > 1) {  in rms_wlock()
1200 		msleep(&rms->owner, &rms->mtx, (PUSER - 1),  in rms_wlock()
1201 		    mtx_name(&rms->mtx), 0);  in rms_wlock()
1202 		MPASS(rms->readers == 0);  in rms_wlock()
1203 		KASSERT(rms->owner == RMS_TRANSIENT,  in rms_wlock()
1205 		    rms->owner));  in rms_wlock()
1209 	KASSERT(rms->owner == RMS_NOOWNER,  in rms_wlock()
1210 	    ("%s: unexpected owner value %p\n", __func__, rms->owner));  in rms_wlock()
1212 	rms_wlock_switch(rms);  in rms_wlock()
1213 	rms_assert_no_pcpu_readers(rms);  in rms_wlock()
1215 	if (rms->readers > 0) {  in rms_wlock()
1216 		msleep(&rms->writers, &rms->mtx, (PUSER - 1),  in rms_wlock()
1217 		    mtx_name(&rms->mtx), 0);  in rms_wlock()
1221 	rms->owner = curthread;  in rms_wlock()
1222 	rms_assert_no_pcpu_readers(rms);  in rms_wlock()
1223 	mtx_unlock(&rms->mtx);  in rms_wlock()
1224 	MPASS(rms->readers == 0);  in rms_wlock()
1229 rms_wunlock(struct rmslock *rms)  in rms_wunlock()  argument
1232 	mtx_lock(&rms->mtx);  in rms_wunlock()
1233 	KASSERT(rms->owner == curthread,  in rms_wunlock()
1234 	    ("%s: unexpected owner value %p\n", __func__, rms->owner));  in rms_wunlock()
1235 	MPASS(rms->writers >= 1);  in rms_wunlock()
1236 	MPASS(rms->readers == 0);  in rms_wunlock()
1237 	rms->writers--;  in rms_wunlock()
1238 	if (rms->writers > 0) {  in rms_wunlock()
1239 		wakeup_one(&rms->owner);  in rms_wunlock()
1240 		rms->owner = RMS_TRANSIENT;  in rms_wunlock()
1242 		wakeup(&rms->readers);  in rms_wunlock()
1243 		rms->owner = RMS_NOOWNER;  in rms_wunlock()
1245 	mtx_unlock(&rms->mtx);  in rms_wunlock()
1250 rms_unlock(struct rmslock *rms)  in rms_unlock()  argument
1253 	if (rms_wowned(rms))  in rms_unlock()
1254 		rms_wunlock(rms);  in rms_unlock()
1256 		rms_runlock(rms);  in rms_unlock()