1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2005 David Xu <davidxu@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #ifndef _THR_FBSD_UMTX_H_ 32 #define _THR_FBSD_UMTX_H_ 33 34 #include <strings.h> 35 #include <sys/umtx.h> 36 37 #ifdef __LP64__ 38 #define DEFAULT_UMUTEX {0,0,{0,0},0,{0,0}} 39 #else 40 #define DEFAULT_UMUTEX {0,0,{0,0},0,0,{0,0}} 41 #endif 42 #define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}} 43 44 int _umtx_op_err(void *, int op, u_long, void *, void *) __hidden; 45 int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden; 46 int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden; 47 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 48 const struct timespec *timeout) __hidden; 49 int __thr_umutex_unlock(struct umutex *mtx) __hidden; 50 int __thr_umutex_trylock(struct umutex *mtx) __hidden; 51 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, 52 uint32_t *oldceiling) __hidden; 53 54 void _thr_umutex_init(struct umutex *mtx) __hidden; 55 void _thr_urwlock_init(struct urwlock *rwl) __hidden; 56 57 int _thr_umtx_wait(volatile long *mtx, long exp, 58 const struct timespec *timeout) __hidden; 59 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp, 60 const struct timespec *timeout, int shared) __hidden; 61 int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid, 62 const struct timespec *timeout, int shared) __hidden; 63 int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden; 64 int _thr_ucond_wait(struct ucond *cv, struct umutex *m, 65 const struct timespec *timeout, int flags) __hidden; 66 void _thr_ucond_init(struct ucond *cv) __hidden; 67 int _thr_ucond_signal(struct ucond *cv) __hidden; 68 int _thr_ucond_broadcast(struct ucond *cv) __hidden; 69 70 int __thr_rwlock_rdlock(struct urwlock *rwlock, int flags, 71 const struct timespec *tsp) __hidden; 72 int __thr_rwlock_wrlock(struct urwlock *rwlock, 73 const struct timespec *tsp) __hidden; 74 int __thr_rwlock_unlock(struct urwlock *rwlock) __hidden; 75 76 /* Internal used only */ 77 void _thr_rwl_rdlock(struct urwlock *rwlock) __hidden; 78 void _thr_rwl_wrlock(struct urwlock *rwlock) __hidden; 79 void _thr_rwl_unlock(struct urwlock *rwlock) __hidden; 80 81 static inline int 82 _thr_umutex_trylock(struct umutex *mtx, uint32_t id) 83 { 84 85 if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) 86 return (0); 87 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) && 88 atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD, 89 id | UMUTEX_CONTESTED)) 90 return (EOWNERDEAD); 91 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV)) 92 return (ENOTRECOVERABLE); 93 if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0) 94 return (EBUSY); 95 return (__thr_umutex_trylock(mtx)); 96 } 97 98 static inline int 99 _thr_umutex_trylock2(struct umutex *mtx, uint32_t id) 100 { 101 102 if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0) 103 return (0); 104 if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED && 105 __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | 106 UMUTEX_PRIO_INHERIT)) == 0) && 107 atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, 108 id | UMUTEX_CONTESTED)) 109 return (0); 110 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) && 111 atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD, 112 id | UMUTEX_CONTESTED)) 113 return (EOWNERDEAD); 114 if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV)) 115 return (ENOTRECOVERABLE); 116 return (EBUSY); 117 } 118 119 static inline int 120 _thr_umutex_lock(struct umutex *mtx, uint32_t id) 121 { 122 123 if (_thr_umutex_trylock2(mtx, id) == 0) 124 return (0); 125 return (__thr_umutex_lock(mtx, id)); 126 } 127 128 static inline int 129 _thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) 130 { 131 132 if (_thr_umutex_trylock2(mtx, id) == 0) 133 return (0); 134 return (__thr_umutex_lock_spin(mtx, id)); 135 } 136 137 static inline int 138 _thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 139 const struct timespec *timeout) 140 { 141 142 if (_thr_umutex_trylock2(mtx, id) == 0) 143 return (0); 144 return (__thr_umutex_timedlock(mtx, id, timeout)); 145 } 146 147 static inline int 148 _thr_umutex_unlock2(struct umutex *mtx, uint32_t id, int *defer) 149 { 150 uint32_t flags, owner; 151 bool noncst; 152 153 flags = mtx->m_flags; 154 noncst = (flags & UMUTEX_NONCONSISTENT) != 0; 155 156 if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) { 157 if (atomic_cmpset_rel_32(&mtx->m_owner, id, noncst ? 158 UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED)) 159 return (0); 160 return (__thr_umutex_unlock(mtx)); 161 } 162 163 do { 164 owner = mtx->m_owner; 165 if (__predict_false((owner & ~UMUTEX_CONTESTED) != id)) 166 return (EPERM); 167 } while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, owner, 168 noncst ? UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED))); 169 if ((owner & UMUTEX_CONTESTED) != 0) { 170 if (defer == NULL || noncst) 171 (void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2, 172 flags, 0, 0); 173 else 174 *defer = 1; 175 } 176 return (0); 177 } 178 179 static inline int 180 _thr_umutex_unlock(struct umutex *mtx, uint32_t id) 181 { 182 183 return (_thr_umutex_unlock2(mtx, id, NULL)); 184 } 185 186 static inline int 187 _thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags) 188 { 189 int32_t state, wrflags; 190 191 if ((flags & URWLOCK_PREFER_READER) != 0 || 192 (rwlock->rw_flags & URWLOCK_PREFER_READER) != 0) 193 wrflags = URWLOCK_WRITE_OWNER; 194 else 195 wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS; 196 state = rwlock->rw_state; 197 while (!(state & wrflags)) { 198 if (__predict_false(URWLOCK_READER_COUNT(state) == 199 URWLOCK_MAX_READERS)) 200 return (EAGAIN); 201 if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1)) 202 return (0); 203 state = rwlock->rw_state; 204 } 205 206 return (EBUSY); 207 } 208 209 static inline int 210 _thr_rwlock_trywrlock(struct urwlock *rwlock) 211 { 212 int32_t state; 213 214 state = rwlock->rw_state; 215 while ((state & URWLOCK_WRITE_OWNER) == 0 && 216 URWLOCK_READER_COUNT(state) == 0) { 217 if (atomic_cmpset_acq_32(&rwlock->rw_state, state, 218 state | URWLOCK_WRITE_OWNER)) 219 return (0); 220 state = rwlock->rw_state; 221 } 222 223 return (EBUSY); 224 } 225 226 static inline int 227 _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp) 228 { 229 230 if (_thr_rwlock_tryrdlock(rwlock, flags) == 0) 231 return (0); 232 return (__thr_rwlock_rdlock(rwlock, flags, tsp)); 233 } 234 235 static inline int 236 _thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp) 237 { 238 239 if (_thr_rwlock_trywrlock(rwlock) == 0) 240 return (0); 241 return (__thr_rwlock_wrlock(rwlock, tsp)); 242 } 243 244 static inline int 245 _thr_rwlock_unlock(struct urwlock *rwlock) 246 { 247 int32_t state; 248 249 state = rwlock->rw_state; 250 if ((state & URWLOCK_WRITE_OWNER) != 0) { 251 if (atomic_cmpset_rel_32(&rwlock->rw_state, 252 URWLOCK_WRITE_OWNER, 0)) 253 return (0); 254 } else { 255 for (;;) { 256 if (__predict_false(URWLOCK_READER_COUNT(state) == 0)) 257 return (EPERM); 258 if (!((state & (URWLOCK_WRITE_WAITERS | 259 URWLOCK_READ_WAITERS)) != 0 && 260 URWLOCK_READER_COUNT(state) == 1)) { 261 if (atomic_cmpset_rel_32(&rwlock->rw_state, 262 state, state - 1)) 263 return (0); 264 state = rwlock->rw_state; 265 } else { 266 break; 267 } 268 } 269 } 270 return (__thr_rwlock_unlock(rwlock)); 271 } 272 #endif 273