xref: /freebsd/lib/libthr/thread/thr_umtx.h (revision 94e3ee44c3581ff37c5e01b5ffe5eb16d30079a7)
1 /*-
2  * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD$
27  */
28 
29 #ifndef _THR_FBSD_UMTX_H_
30 #define _THR_FBSD_UMTX_H_
31 
32 #include <strings.h>
33 #include <sys/umtx.h>
34 
35 #define DEFAULT_UMUTEX	{0,0,{0,0},{0,0,0,0}}
36 #define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}}
37 
38 int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
39 int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden;
40 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
41 	const struct timespec *timeout) __hidden;
42 int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) __hidden;
43 int __thr_umutex_trylock(struct umutex *mtx) __hidden;
44 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
45 	uint32_t *oldceiling) __hidden;
46 
47 void _thr_umutex_init(struct umutex *mtx) __hidden;
48 void _thr_urwlock_init(struct urwlock *rwl) __hidden;
49 
50 int _thr_umtx_wait(volatile long *mtx, long exp,
51 	const struct timespec *timeout) __hidden;
52 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp,
53 	const struct timespec *timeout, int shared) __hidden;
54 int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid,
55 	const struct timespec *timeout, int shared) __hidden;
56 int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden;
57 int _thr_ucond_wait(struct ucond *cv, struct umutex *m,
58         const struct timespec *timeout, int check_unpaking) __hidden;
59 void _thr_ucond_init(struct ucond *cv) __hidden;
60 int _thr_ucond_signal(struct ucond *cv) __hidden;
61 int _thr_ucond_broadcast(struct ucond *cv) __hidden;
62 
63 int __thr_rwlock_rdlock(struct urwlock *rwlock, int flags,
64 	const struct timespec *tsp) __hidden;
65 int __thr_rwlock_wrlock(struct urwlock *rwlock,
66 	const struct timespec *tsp) __hidden;
67 int __thr_rwlock_unlock(struct urwlock *rwlock) __hidden;
68 
69 /* Internal used only */
70 void _thr_rwl_rdlock(struct urwlock *rwlock) __hidden;
71 void _thr_rwl_wrlock(struct urwlock *rwlock) __hidden;
72 void _thr_rwl_unlock(struct urwlock *rwlock) __hidden;
73 
74 static inline int
75 _thr_umutex_trylock(struct umutex *mtx, uint32_t id)
76 {
77     if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
78 	return (0);
79     if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0)
80     	return (EBUSY);
81     return (__thr_umutex_trylock(mtx));
82 }
83 
84 static inline int
85 _thr_umutex_trylock2(struct umutex *mtx, uint32_t id)
86 {
87     if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0)
88 	return (0);
89     if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED &&
90         __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0))
91     	if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED))
92 		return (0);
93     return (EBUSY);
94 }
95 
96 static inline int
97 _thr_umutex_lock(struct umutex *mtx, uint32_t id)
98 {
99     if (_thr_umutex_trylock2(mtx, id) == 0)
100 	return (0);
101     return (__thr_umutex_lock(mtx, id));
102 }
103 
104 static inline int
105 _thr_umutex_lock_spin(struct umutex *mtx, uint32_t id)
106 {
107     if (_thr_umutex_trylock2(mtx, id) == 0)
108 	return (0);
109     return (__thr_umutex_lock_spin(mtx, id));
110 }
111 
112 static inline int
113 _thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
114 	const struct timespec *timeout)
115 {
116     if (_thr_umutex_trylock2(mtx, id) == 0)
117 	return (0);
118     return (__thr_umutex_timedlock(mtx, id, timeout));
119 }
120 
121 static inline int
122 _thr_umutex_unlock(struct umutex *mtx, uint32_t id)
123 {
124     if (atomic_cmpset_rel_32(&mtx->m_owner, id, UMUTEX_UNOWNED))
125 	return (0);
126     return (__thr_umutex_unlock(mtx, id));
127 }
128 
129 static inline int
130 _thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags)
131 {
132 	int32_t state;
133 	int32_t wrflags;
134 
135 	if (flags & URWLOCK_PREFER_READER || rwlock->rw_flags & URWLOCK_PREFER_READER)
136 		wrflags = URWLOCK_WRITE_OWNER;
137 	else
138 		wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS;
139 	state = rwlock->rw_state;
140 	while (!(state & wrflags)) {
141 		if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS))
142 			return (EAGAIN);
143 		if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1))
144 			return (0);
145 		state = rwlock->rw_state;
146 	}
147 
148 	return (EBUSY);
149 }
150 
151 static inline int
152 _thr_rwlock_trywrlock(struct urwlock *rwlock)
153 {
154 	int32_t state;
155 
156 	state = rwlock->rw_state;
157 	while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
158 		if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER))
159 			return (0);
160 		state = rwlock->rw_state;
161 	}
162 
163 	return (EBUSY);
164 }
165 
166 static inline int
167 _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp)
168 {
169 	if (_thr_rwlock_tryrdlock(rwlock, flags) == 0)
170 		return (0);
171 	return (__thr_rwlock_rdlock(rwlock, flags, tsp));
172 }
173 
174 static inline int
175 _thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp)
176 {
177 	if (_thr_rwlock_trywrlock(rwlock) == 0)
178 		return (0);
179 	return (__thr_rwlock_wrlock(rwlock, tsp));
180 }
181 
182 static inline int
183 _thr_rwlock_unlock(struct urwlock *rwlock)
184 {
185 	int32_t state;
186 
187 	state = rwlock->rw_state;
188 	if (state & URWLOCK_WRITE_OWNER) {
189 		if (atomic_cmpset_rel_32(&rwlock->rw_state, URWLOCK_WRITE_OWNER, 0))
190 			return (0);
191 	} else {
192 		for (;;) {
193 			if (__predict_false(URWLOCK_READER_COUNT(state) == 0))
194 				return (EPERM);
195 			if (!((state & (URWLOCK_WRITE_WAITERS |
196 			    URWLOCK_READ_WAITERS)) &&
197 			    URWLOCK_READER_COUNT(state) == 1)) {
198 				if (atomic_cmpset_rel_32(&rwlock->rw_state,
199 				    state, state-1))
200 					return (0);
201 				state = rwlock->rw_state;
202 			} else {
203 				break;
204 			}
205 		}
206     	}
207     	return (__thr_rwlock_unlock(rwlock));
208 }
209 #endif
210