1 /* 2 * Copyright (c) 2005 David Xu <davidxu@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "thr_private.h" 31 #include "thr_umtx.h" 32 33 #ifndef HAS__UMTX_OP_ERR 34 int _umtx_op_err(void *obj, int op, u_long val, void *uaddr, void *uaddr2) 35 { 36 if (_umtx_op(obj, op, val, uaddr, uaddr2) == -1) 37 return (errno); 38 return (0); 39 } 40 #endif 41 42 void 43 _thr_umutex_init(struct umutex *mtx) 44 { 45 static const struct umutex default_mtx = DEFAULT_UMUTEX; 46 47 *mtx = default_mtx; 48 } 49 50 void 51 _thr_urwlock_init(struct urwlock *rwl) 52 { 53 static const struct urwlock default_rwl = DEFAULT_URWLOCK; 54 55 *rwl = default_rwl; 56 } 57 58 int 59 __thr_umutex_lock(struct umutex *mtx, uint32_t id) 60 { 61 uint32_t owner; 62 63 if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 64 for (;;) { 65 /* wait in kernel */ 66 _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); 67 68 owner = mtx->m_owner; 69 if ((owner & ~UMUTEX_CONTESTED) == 0 && 70 atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) 71 return (0); 72 } 73 } 74 75 return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); 76 } 77 78 #define SPINLOOPS 1000 79 80 int 81 __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) 82 { 83 uint32_t owner; 84 85 if (!_thr_is_smp) 86 return __thr_umutex_lock(mtx, id); 87 88 if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 89 for (;;) { 90 int count = SPINLOOPS; 91 while (count--) { 92 owner = mtx->m_owner; 93 if ((owner & ~UMUTEX_CONTESTED) == 0) { 94 if (atomic_cmpset_acq_32( 95 &mtx->m_owner, 96 owner, id|owner)) { 97 return (0); 98 } 99 } 100 CPU_SPINWAIT; 101 } 102 103 /* wait in kernel */ 104 _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); 105 } 106 } 107 108 return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); 109 } 110 111 int 112 __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 113 const struct timespec *abstime) 114 { 115 struct _umtx_time *tm_p, timeout; 116 size_t tm_size; 117 uint32_t owner; 118 int ret; 119 120 if (abstime == NULL) { 121 tm_p = NULL; 122 tm_size = 0; 123 } else { 124 timeout._clockid = CLOCK_REALTIME; 125 timeout._flags = UMTX_ABSTIME; 126 timeout._timeout = *abstime; 127 tm_p = &timeout; 128 tm_size = sizeof(timeout); 129 } 130 131 for (;;) { 132 if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 133 134 /* wait in kernel */ 135 ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 136 (void *)tm_size, __DECONST(void *, tm_p)); 137 138 /* now try to lock it */ 139 owner = mtx->m_owner; 140 if ((owner & ~UMUTEX_CONTESTED) == 0 && 141 atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) 142 return (0); 143 } else { 144 ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 145 (void *)tm_size, __DECONST(void *, tm_p)); 146 if (ret == 0) 147 break; 148 } 149 if (ret == ETIMEDOUT) 150 break; 151 } 152 return (ret); 153 } 154 155 int 156 __thr_umutex_unlock(struct umutex *mtx, uint32_t id) 157 { 158 return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0); 159 } 160 161 int 162 __thr_umutex_trylock(struct umutex *mtx) 163 { 164 return _umtx_op_err(mtx, UMTX_OP_MUTEX_TRYLOCK, 0, 0, 0); 165 } 166 167 int 168 __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, 169 uint32_t *oldceiling) 170 { 171 return _umtx_op_err(mtx, UMTX_OP_SET_CEILING, ceiling, oldceiling, 0); 172 } 173 174 int 175 _thr_umtx_wait(volatile long *mtx, long id, const struct timespec *timeout) 176 { 177 if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 178 timeout->tv_nsec <= 0))) 179 return (ETIMEDOUT); 180 return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAIT, id, 0, 181 __DECONST(void*, timeout)); 182 } 183 184 int 185 _thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout, int shared) 186 { 187 if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 188 timeout->tv_nsec <= 0))) 189 return (ETIMEDOUT); 190 return _umtx_op_err(__DEVOLATILE(void *, mtx), 191 shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 0, 192 __DECONST(void*, timeout)); 193 } 194 195 int 196 _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int id, int clockid, 197 const struct timespec *abstime, int shared) 198 { 199 struct _umtx_time *tm_p, timeout; 200 size_t tm_size; 201 202 if (abstime == NULL) { 203 tm_p = NULL; 204 tm_size = 0; 205 } else { 206 timeout._clockid = clockid; 207 timeout._flags = UMTX_ABSTIME; 208 timeout._timeout = *abstime; 209 tm_p = &timeout; 210 tm_size = sizeof(timeout); 211 } 212 213 return _umtx_op_err(__DEVOLATILE(void *, mtx), 214 shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 215 (void *)tm_size, __DECONST(void *, tm_p)); 216 } 217 218 int 219 _thr_umtx_wake(volatile void *mtx, int nr_wakeup, int shared) 220 { 221 return _umtx_op_err(__DEVOLATILE(void *, mtx), shared ? UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE, 222 nr_wakeup, 0, 0); 223 } 224 225 void 226 _thr_ucond_init(struct ucond *cv) 227 { 228 bzero(cv, sizeof(struct ucond)); 229 } 230 231 int 232 _thr_ucond_wait(struct ucond *cv, struct umutex *m, 233 const struct timespec *timeout, int flags) 234 { 235 if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 236 timeout->tv_nsec <= 0))) { 237 struct pthread *curthread = _get_curthread(); 238 _thr_umutex_unlock(m, TID(curthread)); 239 return (ETIMEDOUT); 240 } 241 return _umtx_op_err(cv, UMTX_OP_CV_WAIT, flags, 242 m, __DECONST(void*, timeout)); 243 } 244 245 int 246 _thr_ucond_signal(struct ucond *cv) 247 { 248 if (!cv->c_has_waiters) 249 return (0); 250 return _umtx_op_err(cv, UMTX_OP_CV_SIGNAL, 0, NULL, NULL); 251 } 252 253 int 254 _thr_ucond_broadcast(struct ucond *cv) 255 { 256 if (!cv->c_has_waiters) 257 return (0); 258 return _umtx_op_err(cv, UMTX_OP_CV_BROADCAST, 0, NULL, NULL); 259 } 260 261 int 262 __thr_rwlock_rdlock(struct urwlock *rwlock, int flags, 263 const struct timespec *tsp) 264 { 265 struct _umtx_time timeout, *tm_p; 266 size_t tm_size; 267 268 if (tsp == NULL) { 269 tm_p = NULL; 270 tm_size = 0; 271 } else { 272 timeout._timeout = *tsp; 273 timeout._flags = UMTX_ABSTIME; 274 timeout._clockid = CLOCK_REALTIME; 275 tm_p = &timeout; 276 tm_size = sizeof(timeout); 277 } 278 return _umtx_op_err(rwlock, UMTX_OP_RW_RDLOCK, flags, (void *)tm_size, tm_p); 279 } 280 281 int 282 __thr_rwlock_wrlock(struct urwlock *rwlock, const struct timespec *tsp) 283 { 284 struct _umtx_time timeout, *tm_p; 285 size_t tm_size; 286 287 if (tsp == NULL) { 288 tm_p = NULL; 289 tm_size = 0; 290 } else { 291 timeout._timeout = *tsp; 292 timeout._flags = UMTX_ABSTIME; 293 timeout._clockid = CLOCK_REALTIME; 294 tm_p = &timeout; 295 tm_size = sizeof(timeout); 296 } 297 return _umtx_op_err(rwlock, UMTX_OP_RW_WRLOCK, 0, (void *)tm_size, tm_p); 298 } 299 300 int 301 __thr_rwlock_unlock(struct urwlock *rwlock) 302 { 303 return _umtx_op_err(rwlock, UMTX_OP_RW_UNLOCK, 0, NULL, NULL); 304 } 305 306 void 307 _thr_rwl_rdlock(struct urwlock *rwlock) 308 { 309 int ret; 310 311 for (;;) { 312 if (_thr_rwlock_tryrdlock(rwlock, URWLOCK_PREFER_READER) == 0) 313 return; 314 ret = __thr_rwlock_rdlock(rwlock, URWLOCK_PREFER_READER, NULL); 315 if (ret == 0) 316 return; 317 if (ret != EINTR) 318 PANIC("rdlock error"); 319 } 320 } 321 322 void 323 _thr_rwl_wrlock(struct urwlock *rwlock) 324 { 325 int ret; 326 327 for (;;) { 328 if (_thr_rwlock_trywrlock(rwlock) == 0) 329 return; 330 ret = __thr_rwlock_wrlock(rwlock, NULL); 331 if (ret == 0) 332 return; 333 if (ret != EINTR) 334 PANIC("wrlock error"); 335 } 336 } 337 338 void 339 _thr_rwl_unlock(struct urwlock *rwlock) 340 { 341 if (_thr_rwlock_unlock(rwlock)) 342 PANIC("unlock error"); 343 } 344