1 /*- 2 * Copyright (c) 2005 David Xu <davidxu@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _THR_FBSD_UMTX_H_ 30 #define _THR_FBSD_UMTX_H_ 31 32 #include <strings.h> 33 #include <sys/umtx.h> 34 35 #define DEFAULT_UMUTEX {0,0,{0,0},{0,0,0,0}} 36 #define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}} 37 38 int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden; 39 int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden; 40 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 41 const struct timespec *timeout) __hidden; 42 int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) __hidden; 43 int __thr_umutex_trylock(struct umutex *mtx) __hidden; 44 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, 45 uint32_t *oldceiling) __hidden; 46 47 void _thr_umutex_init(struct umutex *mtx) __hidden; 48 void _thr_urwlock_init(struct urwlock *rwl) __hidden; 49 50 int _thr_umtx_wait(volatile long *mtx, long exp, 51 const struct timespec *timeout) __hidden; 52 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp, 53 const struct timespec *timeout, int shared) __hidden; 54 int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid, 55 const struct timespec *timeout, int shared) __hidden; 56 int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden; 57 int _thr_ucond_wait(struct ucond *cv, struct umutex *m, 58 const struct timespec *timeout, int check_unpaking) __hidden; 59 void _thr_ucond_init(struct ucond *cv) __hidden; 60 int _thr_ucond_signal(struct ucond *cv) __hidden; 61 int _thr_ucond_broadcast(struct ucond *cv) __hidden; 62 63 int __thr_rwlock_rdlock(struct urwlock *rwlock, int flags, 64 const struct timespec *tsp) __hidden; 65 int __thr_rwlock_wrlock(struct urwlock *rwlock, 66 const struct timespec *tsp) __hidden; 67 int __thr_rwlock_unlock(struct urwlock *rwlock) __hidden; 68 69 /* Internal used only */ 70 void _thr_rwl_rdlock(struct urwlock *rwlock) __hidden; 71 void _thr_rwl_wrlock(struct urwlock *rwlock) __hidden; 72 void _thr_rwl_unlock(struct urwlock *rwlock) __hidden; 73 74 static inline int 75 _thr_umutex_trylock(struct umutex *mtx, uint32_t id) 76 { 77 if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) 78 return (0); 79 if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0) 80 return (EBUSY); 81 return (__thr_umutex_trylock(mtx)); 82 } 83 84 static inline int 85 _thr_umutex_trylock2(struct umutex *mtx, uint32_t id) 86 { 87 if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0) 88 return (0); 89 if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED && 90 __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0)) 91 if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED)) 92 return (0); 93 return (EBUSY); 94 } 95 96 static inline int 97 _thr_umutex_lock(struct umutex *mtx, uint32_t id) 98 { 99 if (_thr_umutex_trylock2(mtx, id) == 0) 100 return (0); 101 return (__thr_umutex_lock(mtx, id)); 102 } 103 104 static inline int 105 _thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) 106 { 107 if (_thr_umutex_trylock2(mtx, id) == 0) 108 return (0); 109 return (__thr_umutex_lock_spin(mtx, id)); 110 } 111 112 static inline int 113 _thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 114 const struct timespec *timeout) 115 { 116 if (_thr_umutex_trylock2(mtx, id) == 0) 117 return (0); 118 return (__thr_umutex_timedlock(mtx, id, timeout)); 119 } 120 121 static inline int 122 _thr_umutex_unlock(struct umutex *mtx, uint32_t id) 123 { 124 if (atomic_cmpset_rel_32(&mtx->m_owner, id, UMUTEX_UNOWNED)) 125 return (0); 126 return (__thr_umutex_unlock(mtx, id)); 127 } 128 129 static inline int 130 _thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags) 131 { 132 int32_t state; 133 int32_t wrflags; 134 135 if (flags & URWLOCK_PREFER_READER || rwlock->rw_flags & URWLOCK_PREFER_READER) 136 wrflags = URWLOCK_WRITE_OWNER; 137 else 138 wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS; 139 state = rwlock->rw_state; 140 while (!(state & wrflags)) { 141 if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) 142 return (EAGAIN); 143 if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1)) 144 return (0); 145 state = rwlock->rw_state; 146 } 147 148 return (EBUSY); 149 } 150 151 static inline int 152 _thr_rwlock_trywrlock(struct urwlock *rwlock) 153 { 154 int32_t state; 155 156 state = rwlock->rw_state; 157 while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { 158 if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER)) 159 return (0); 160 state = rwlock->rw_state; 161 } 162 163 return (EBUSY); 164 } 165 166 static inline int 167 _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp) 168 { 169 if (_thr_rwlock_tryrdlock(rwlock, flags) == 0) 170 return (0); 171 return (__thr_rwlock_rdlock(rwlock, flags, tsp)); 172 } 173 174 static inline int 175 _thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp) 176 { 177 if (_thr_rwlock_trywrlock(rwlock) == 0) 178 return (0); 179 return (__thr_rwlock_wrlock(rwlock, tsp)); 180 } 181 182 static inline int 183 _thr_rwlock_unlock(struct urwlock *rwlock) 184 { 185 int32_t state; 186 187 state = rwlock->rw_state; 188 if (state & URWLOCK_WRITE_OWNER) { 189 if (atomic_cmpset_rel_32(&rwlock->rw_state, URWLOCK_WRITE_OWNER, 0)) 190 return (0); 191 } else { 192 for (;;) { 193 if (__predict_false(URWLOCK_READER_COUNT(state) == 0)) 194 return (EPERM); 195 if (!((state & (URWLOCK_WRITE_WAITERS | 196 URWLOCK_READ_WAITERS)) && 197 URWLOCK_READER_COUNT(state) == 1)) { 198 if (atomic_cmpset_rel_32(&rwlock->rw_state, 199 state, state-1)) 200 return (0); 201 state = rwlock->rw_state; 202 } else { 203 break; 204 } 205 } 206 } 207 return (__thr_rwlock_unlock(rwlock)); 208 } 209 #endif 210