xref: /freebsd/lib/libthr/thread/thr_umtx.h (revision 62cfcf62f627e5093fb37026a6d8c98e4d2ef04c)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #ifndef _THR_FBSD_UMTX_H_
32 #define _THR_FBSD_UMTX_H_
33 
34 #include <strings.h>
35 #include <sys/umtx.h>
36 
37 #ifdef __LP64__
38 #define DEFAULT_UMUTEX	{0,0,{0,0},0,{0,0}}
39 #else
40 #define DEFAULT_UMUTEX	{0,0,{0,0},0,0,{0,0}}
41 #endif
42 #define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}}
43 
44 int _umtx_op_err(void *, int op, u_long, void *, void *) __hidden;
45 int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
46 int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden;
47 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
48 	const struct timespec *timeout) __hidden;
49 int __thr_umutex_unlock(struct umutex *mtx) __hidden;
50 int __thr_umutex_trylock(struct umutex *mtx) __hidden;
51 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
52 	uint32_t *oldceiling) __hidden;
53 
54 void _thr_umutex_init(struct umutex *mtx) __hidden;
55 void _thr_urwlock_init(struct urwlock *rwl) __hidden;
56 
57 int _thr_umtx_wait(volatile long *mtx, long exp,
58 	const struct timespec *timeout) __hidden;
59 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp,
60 	const struct timespec *timeout, int shared) __hidden;
61 int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid,
62 	const struct timespec *timeout, int shared) __hidden;
63 int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden;
64 int _thr_ucond_wait(struct ucond *cv, struct umutex *m,
65         const struct timespec *timeout, int flags) __hidden;
66 void _thr_ucond_init(struct ucond *cv) __hidden;
67 int _thr_ucond_signal(struct ucond *cv) __hidden;
68 int _thr_ucond_broadcast(struct ucond *cv) __hidden;
69 
70 int __thr_rwlock_rdlock(struct urwlock *rwlock, int flags,
71 	const struct timespec *tsp) __hidden;
72 int __thr_rwlock_wrlock(struct urwlock *rwlock,
73 	const struct timespec *tsp) __hidden;
74 int __thr_rwlock_unlock(struct urwlock *rwlock) __hidden;
75 
76 /* Internal used only */
77 void _thr_rwl_rdlock(struct urwlock *rwlock) __hidden;
78 void _thr_rwl_wrlock(struct urwlock *rwlock) __hidden;
79 void _thr_rwl_unlock(struct urwlock *rwlock) __hidden;
80 
81 static inline int
82 _thr_umutex_trylock(struct umutex *mtx, uint32_t id)
83 {
84 
85 	if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
86 		return (0);
87 	if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) &&
88 	    atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD,
89 	    id | UMUTEX_CONTESTED))
90 		return (EOWNERDEAD);
91 	if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV))
92 		return (ENOTRECOVERABLE);
93 	if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0)
94 		return (EBUSY);
95 	return (__thr_umutex_trylock(mtx));
96 }
97 
98 static inline int
99 _thr_umutex_trylock2(struct umutex *mtx, uint32_t id)
100 {
101 
102 	if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0)
103 		return (0);
104 	if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED &&
105 	    __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT |
106 	   UMUTEX_PRIO_INHERIT)) == 0) &&
107 	   atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED,
108 	   id | UMUTEX_CONTESTED))
109 		return (0);
110 	if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) &&
111 	    atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD,
112 	    id | UMUTEX_CONTESTED))
113 		return (EOWNERDEAD);
114 	if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV))
115 		return (ENOTRECOVERABLE);
116 	return (EBUSY);
117 }
118 
119 static inline int
120 _thr_umutex_lock(struct umutex *mtx, uint32_t id)
121 {
122 
123 	if (_thr_umutex_trylock2(mtx, id) == 0)
124 		return (0);
125 	return (__thr_umutex_lock(mtx, id));
126 }
127 
128 static inline int
129 _thr_umutex_lock_spin(struct umutex *mtx, uint32_t id)
130 {
131 
132 	if (_thr_umutex_trylock2(mtx, id) == 0)
133 		return (0);
134 	return (__thr_umutex_lock_spin(mtx, id));
135 }
136 
137 static inline int
138 _thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
139     const struct timespec *timeout)
140 {
141 
142 	if (_thr_umutex_trylock2(mtx, id) == 0)
143 		return (0);
144 	return (__thr_umutex_timedlock(mtx, id, timeout));
145 }
146 
147 static inline int
148 _thr_umutex_unlock2(struct umutex *mtx, uint32_t id, int *defer)
149 {
150 	uint32_t flags, owner;
151 	bool noncst;
152 
153 	flags = mtx->m_flags;
154 	noncst = (flags & UMUTEX_NONCONSISTENT) != 0;
155 
156 	if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) {
157 		if (atomic_cmpset_rel_32(&mtx->m_owner, id, noncst ?
158 		    UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED))
159 			return (0);
160 		return (__thr_umutex_unlock(mtx));
161 	}
162 
163 	do {
164 		owner = mtx->m_owner;
165 		if (__predict_false((owner & ~UMUTEX_CONTESTED) != id))
166 			return (EPERM);
167 	} while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, owner,
168 	    noncst ? UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED)));
169 	if ((owner & UMUTEX_CONTESTED) != 0) {
170 		if (defer == NULL || noncst)
171 			(void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2,
172 			    flags, 0, 0);
173 		else
174 			*defer = 1;
175 	}
176 	return (0);
177 }
178 
179 static inline int
180 _thr_umutex_unlock(struct umutex *mtx, uint32_t id)
181 {
182 
183 	return (_thr_umutex_unlock2(mtx, id, NULL));
184 }
185 
186 static inline int
187 _thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags)
188 {
189 	int32_t state, wrflags;
190 
191 	if ((flags & URWLOCK_PREFER_READER) != 0 ||
192 	    (rwlock->rw_flags & URWLOCK_PREFER_READER) != 0)
193 		wrflags = URWLOCK_WRITE_OWNER;
194 	else
195 		wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS;
196 	state = rwlock->rw_state;
197 	while (!(state & wrflags)) {
198 		if (__predict_false(URWLOCK_READER_COUNT(state) ==
199 		    URWLOCK_MAX_READERS))
200 			return (EAGAIN);
201 		if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1))
202 			return (0);
203 		state = rwlock->rw_state;
204 	}
205 
206 	return (EBUSY);
207 }
208 
209 static inline int
210 _thr_rwlock_trywrlock(struct urwlock *rwlock)
211 {
212 	int32_t state;
213 
214 	state = rwlock->rw_state;
215 	while ((state & URWLOCK_WRITE_OWNER) == 0 &&
216 	    URWLOCK_READER_COUNT(state) == 0) {
217 		if (atomic_cmpset_acq_32(&rwlock->rw_state, state,
218 		    state | URWLOCK_WRITE_OWNER))
219 			return (0);
220 		state = rwlock->rw_state;
221 	}
222 
223 	return (EBUSY);
224 }
225 
226 static inline int
227 _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp)
228 {
229 
230 	if (_thr_rwlock_tryrdlock(rwlock, flags) == 0)
231 		return (0);
232 	return (__thr_rwlock_rdlock(rwlock, flags, tsp));
233 }
234 
235 static inline int
236 _thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp)
237 {
238 
239 	if (_thr_rwlock_trywrlock(rwlock) == 0)
240 		return (0);
241 	return (__thr_rwlock_wrlock(rwlock, tsp));
242 }
243 
244 static inline int
245 _thr_rwlock_unlock(struct urwlock *rwlock)
246 {
247 	int32_t state;
248 
249 	state = rwlock->rw_state;
250 	if ((state & URWLOCK_WRITE_OWNER) != 0) {
251 		if (atomic_cmpset_rel_32(&rwlock->rw_state,
252 		    URWLOCK_WRITE_OWNER, 0))
253 			return (0);
254 	} else {
255 		for (;;) {
256 			if (__predict_false(URWLOCK_READER_COUNT(state) == 0))
257 				return (EPERM);
258 			if (!((state & (URWLOCK_WRITE_WAITERS |
259 			    URWLOCK_READ_WAITERS)) != 0 &&
260 			    URWLOCK_READER_COUNT(state) == 1)) {
261 				if (atomic_cmpset_rel_32(&rwlock->rw_state,
262 				    state, state - 1))
263 					return (0);
264 				state = rwlock->rw_state;
265 			} else {
266 				break;
267 			}
268 		}
269     	}
270     	return (__thr_rwlock_unlock(rwlock));
271 }
272 #endif
273