1 /*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. 5 * Copyright (c) 2006 David Xu <davidxu@freebsd.org>. 6 * Copyright (c) 2015, 2016 The FreeBSD Foundation 7 * 8 * All rights reserved. 9 * 10 * Portions of this software were developed by Konstantin Belousov 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by John Birrell. 24 * 4. Neither the name of the author nor the names of any co-contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include "namespace.h" 42 #include <stdlib.h> 43 #include <errno.h> 44 #include <string.h> 45 #include <sys/param.h> 46 #include <sys/queue.h> 47 #include <pthread.h> 48 #include <pthread_np.h> 49 #include "un-namespace.h" 50 51 #include "thr_private.h" 52 53 _Static_assert(sizeof(struct pthread_mutex) <= THR_PAGE_SIZE_MIN, 54 "pthread_mutex is too large for off-page"); 55 56 /* 57 * Keep the thread-private linkage off the cache line holding the lock word; 58 * Only meaningful where cache line is 64 bytes. 59 */ 60 #if defined(__LP64__) && CACHE_LINE_SIZE == 64 61 _Static_assert(offsetof(struct pthread_mutex, m_qe) >= CACHE_LINE_SIZE, 62 "m_qe must not share a cache line with m_lock.m_owner"); 63 #endif 64 65 /* 66 * Prototypes 67 */ 68 int __pthread_mutex_timedlock(pthread_mutex_t * __restrict mutex, 69 const struct timespec * __restrict abstime); 70 int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count); 71 int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 72 int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 73 int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 74 int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count); 75 int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 76 77 static int mutex_self_trylock(pthread_mutex_t); 78 static int mutex_self_lock(pthread_mutex_t, 79 const struct timespec *abstime); 80 static int mutex_unlock_common(struct pthread_mutex *, bool, int *); 81 static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, 82 const struct timespec *); 83 static void mutex_init_robust(struct pthread *curthread); 84 static int mutex_qidx(struct pthread_mutex *m); 85 static bool is_robust_mutex(struct pthread_mutex *m); 86 static bool is_pshared_mutex(struct pthread_mutex *m); 87 88 __weak_reference(__Tthr_mutex_init, pthread_mutex_init); 89 __weak_reference(__Tthr_mutex_init, __pthread_mutex_init); 90 __strong_reference(__Tthr_mutex_init, _pthread_mutex_init); 91 __weak_reference(__Tthr_mutex_lock, pthread_mutex_lock); 92 __weak_reference(__Tthr_mutex_lock, __pthread_mutex_lock); 93 __strong_reference(__Tthr_mutex_lock, _pthread_mutex_lock); 94 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); 95 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock); 96 __weak_reference(__Tthr_mutex_trylock, pthread_mutex_trylock); 97 __weak_reference(__Tthr_mutex_trylock, __pthread_mutex_trylock); 98 __strong_reference(__Tthr_mutex_trylock, _pthread_mutex_trylock); 99 __weak_reference(_Tthr_mutex_consistent, pthread_mutex_consistent); 100 __weak_reference(_Tthr_mutex_consistent, _pthread_mutex_consistent); 101 __strong_reference(_Tthr_mutex_consistent, __pthread_mutex_consistent); 102 103 /* Single underscore versions provided for libc internal usage: */ 104 /* No difference between libc and application usage of these: */ 105 __weak_reference(_thr_mutex_destroy, pthread_mutex_destroy); 106 __weak_reference(_thr_mutex_destroy, _pthread_mutex_destroy); 107 __weak_reference(_thr_mutex_unlock, pthread_mutex_unlock); 108 __weak_reference(_thr_mutex_unlock, _pthread_mutex_unlock); 109 110 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling); 111 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling); 112 113 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np); 114 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np); 115 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np); 116 117 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np); 118 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np); 119 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np); 120 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np); 121 122 static void 123 mutex_init_link(struct pthread_mutex *m __unused) 124 { 125 126 #if defined(_PTHREADS_INVARIANTS) 127 m->m_qe.tqe_prev = NULL; 128 m->m_qe.tqe_next = NULL; 129 m->m_pqe.tqe_prev = NULL; 130 m->m_pqe.tqe_next = NULL; 131 #endif 132 } 133 134 static void 135 mutex_assert_is_owned(struct pthread_mutex *m __unused) 136 { 137 138 #if defined(_PTHREADS_INVARIANTS) 139 if (__predict_false(m->m_qe.tqe_prev == NULL)) 140 PANIC("mutex %p own %#x is not on list %p %p", 141 m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); 142 #endif 143 } 144 145 static void 146 mutex_assert_not_owned(struct pthread *curthread __unused, 147 struct pthread_mutex *m __unused) 148 { 149 150 #if defined(_PTHREADS_INVARIANTS) 151 if (__predict_false(m->m_qe.tqe_prev != NULL || 152 m->m_qe.tqe_next != NULL)) 153 PANIC("mutex %p own %#x is on list %p %p", 154 m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); 155 if (__predict_false(is_robust_mutex(m) && 156 (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL || 157 (is_pshared_mutex(m) && curthread->robust_list == 158 (uintptr_t)&m->m_lock) || 159 (!is_pshared_mutex(m) && curthread->priv_robust_list == 160 (uintptr_t)&m->m_lock)))) 161 PANIC( 162 "mutex %p own %#x is on robust linkage %p %p head %p phead %p", 163 m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk, 164 m->m_rb_prev, (void *)curthread->robust_list, 165 (void *)curthread->priv_robust_list); 166 #endif 167 } 168 169 static bool 170 is_pshared_mutex(struct pthread_mutex *m) 171 { 172 173 return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0); 174 } 175 176 static bool 177 is_robust_mutex(struct pthread_mutex *m) 178 { 179 180 return ((m->m_lock.m_flags & UMUTEX_ROBUST) != 0); 181 } 182 183 int 184 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m) 185 { 186 187 #if defined(_PTHREADS_INVARIANTS) 188 if (__predict_false(curthread->inact_mtx != 0)) 189 PANIC("inact_mtx enter"); 190 #endif 191 if (!is_robust_mutex(m)) 192 return (0); 193 194 mutex_init_robust(curthread); 195 curthread->inact_mtx = (uintptr_t)&m->m_lock; 196 return (1); 197 } 198 199 void 200 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused) 201 { 202 203 #if defined(_PTHREADS_INVARIANTS) 204 if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock)) 205 PANIC("inact_mtx leave"); 206 #endif 207 curthread->inact_mtx = 0; 208 } 209 210 static int 211 mutex_check_attr(const struct pthread_mutex_attr *attr) 212 { 213 214 if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK || 215 attr->m_type >= PTHREAD_MUTEX_TYPE_MAX) 216 return (EINVAL); 217 if (attr->m_protocol < PTHREAD_PRIO_NONE || 218 attr->m_protocol > PTHREAD_PRIO_PROTECT) 219 return (EINVAL); 220 return (0); 221 } 222 223 static void 224 mutex_init_robust(struct pthread *curthread) 225 { 226 struct umtx_robust_lists_params rb; 227 228 if (curthread == NULL) 229 curthread = _get_curthread(); 230 if (curthread->robust_inited) 231 return; 232 rb.robust_list_offset = (uintptr_t)&curthread->robust_list; 233 rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list; 234 rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx; 235 _umtx_op(NULL, UMTX_OP_ROBUST_LISTS, sizeof(rb), &rb, NULL); 236 curthread->robust_inited = 1; 237 } 238 239 static void 240 mutex_init_body(struct pthread_mutex *pmutex, 241 const struct pthread_mutex_attr *attr) 242 { 243 244 pmutex->m_flags = attr->m_type; 245 pmutex->m_count = 0; 246 pmutex->m_spinloops = 0; 247 pmutex->m_yieldloops = 0; 248 mutex_init_link(pmutex); 249 switch (attr->m_protocol) { 250 case PTHREAD_PRIO_NONE: 251 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 252 pmutex->m_lock.m_flags = 0; 253 break; 254 case PTHREAD_PRIO_INHERIT: 255 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 256 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT; 257 break; 258 case PTHREAD_PRIO_PROTECT: 259 pmutex->m_lock.m_owner = UMUTEX_CONTESTED; 260 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT; 261 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling; 262 break; 263 } 264 if (attr->m_pshared == PTHREAD_PROCESS_SHARED) 265 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED; 266 if (attr->m_robust == PTHREAD_MUTEX_ROBUST) { 267 mutex_init_robust(NULL); 268 pmutex->m_lock.m_flags |= UMUTEX_ROBUST; 269 } 270 if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) { 271 pmutex->m_spinloops = _thr_spinloops; 272 pmutex->m_yieldloops = _thr_yieldloops; 273 } 274 } 275 276 static int 277 mutex_init(pthread_mutex_t *mutex, 278 const struct pthread_mutex_attr *mutex_attr, 279 void *(calloc_cb)(size_t, size_t)) 280 { 281 const struct pthread_mutex_attr *attr; 282 struct pthread_mutex *pmutex; 283 int error; 284 285 if (mutex_attr == NULL) { 286 attr = &_pthread_mutexattr_default; 287 } else { 288 attr = mutex_attr; 289 error = mutex_check_attr(attr); 290 if (error != 0) 291 return (error); 292 } 293 if ((pmutex = (pthread_mutex_t)calloc_cb(1, 294 sizeof(struct pthread_mutex))) == NULL) 295 return (ENOMEM); 296 mutex_init_body(pmutex, attr); 297 *mutex = pmutex; 298 return (0); 299 } 300 301 static int 302 init_static(struct pthread *thread, pthread_mutex_t *mutex) 303 { 304 int ret; 305 306 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 307 308 if (*mutex == THR_MUTEX_INITIALIZER) 309 ret = mutex_init(mutex, &_pthread_mutexattr_default, 310 __thr_calloc); 311 else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) 312 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, 313 __thr_calloc); 314 else 315 ret = 0; 316 THR_LOCK_RELEASE(thread, &_mutex_static_lock); 317 318 return (ret); 319 } 320 321 static void 322 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) 323 { 324 struct pthread_mutex *m2; 325 326 m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue); 327 if (m2 != NULL) 328 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0]; 329 else 330 m->m_lock.m_ceilings[1] = -1; 331 } 332 333 static void 334 shared_mutex_init(struct pthread_mutex *pmtx, const struct 335 pthread_mutex_attr *mutex_attr) 336 { 337 static const struct pthread_mutex_attr foobar_mutex_attr = { 338 .m_type = PTHREAD_MUTEX_DEFAULT, 339 .m_protocol = PTHREAD_PRIO_NONE, 340 .m_ceiling = 0, 341 .m_pshared = PTHREAD_PROCESS_SHARED, 342 .m_robust = PTHREAD_MUTEX_STALLED, 343 }; 344 bool done; 345 346 /* 347 * Hack to allow multiple pthread_mutex_init() calls on the 348 * same process-shared mutex. We rely on kernel allocating 349 * zeroed offpage for the mutex, i.e. the 350 * PMUTEX_INITSTAGE_ALLOC value must be zero. 351 */ 352 for (done = false; !done;) { 353 switch (pmtx->m_ps) { 354 case PMUTEX_INITSTAGE_DONE: 355 atomic_thread_fence_acq(); 356 done = true; 357 break; 358 case PMUTEX_INITSTAGE_ALLOC: 359 if (atomic_cmpset_int(&pmtx->m_ps, 360 PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) { 361 if (mutex_attr == NULL) 362 mutex_attr = &foobar_mutex_attr; 363 mutex_init_body(pmtx, mutex_attr); 364 atomic_store_rel_int(&pmtx->m_ps, 365 PMUTEX_INITSTAGE_DONE); 366 done = true; 367 } 368 break; 369 case PMUTEX_INITSTAGE_BUSY: 370 _pthread_yield(); 371 break; 372 default: 373 PANIC("corrupted offpage"); 374 break; 375 } 376 } 377 } 378 379 int 380 __Tthr_mutex_init(pthread_mutex_t * __restrict mutex, 381 const pthread_mutexattr_t * __restrict mutex_attr) 382 { 383 struct pthread_mutex *pmtx; 384 int ret; 385 386 _thr_check_init(); 387 388 if (mutex_attr != NULL) { 389 ret = mutex_check_attr(*mutex_attr); 390 if (ret != 0) 391 return (ret); 392 } 393 if (mutex_attr == NULL || 394 (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { 395 __thr_malloc_init(); 396 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, 397 __thr_calloc)); 398 } 399 pmtx = __thr_pshared_offpage(__DECONST(void *, mutex), 1); 400 if (pmtx == NULL) 401 return (EFAULT); 402 *mutex = THR_PSHARED_PTR; 403 shared_mutex_init(pmtx, *mutex_attr); 404 return (0); 405 } 406 407 /* This function is used internally by malloc. */ 408 int 409 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 410 void *(calloc_cb)(size_t, size_t)) 411 { 412 static const struct pthread_mutex_attr attr = { 413 .m_type = PTHREAD_MUTEX_NORMAL, 414 .m_protocol = PTHREAD_PRIO_NONE, 415 .m_ceiling = 0, 416 .m_pshared = PTHREAD_PROCESS_PRIVATE, 417 .m_robust = PTHREAD_MUTEX_STALLED, 418 }; 419 int ret; 420 421 ret = mutex_init(mutex, &attr, calloc_cb); 422 if (ret == 0) 423 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE; 424 return (ret); 425 } 426 427 /* 428 * Fix mutex ownership for child process. 429 * 430 * Process private mutex ownership is transmitted from the forking 431 * thread to the child process. 432 * 433 * Process shared mutex should not be inherited because owner is 434 * forking thread which is in parent process, they are removed from 435 * the owned mutex list. 436 */ 437 static void 438 queue_fork(struct pthread *curthread, struct mutex_queue *q, 439 struct mutex_queue *qp, uint bit) 440 { 441 struct pthread_mutex *m; 442 443 TAILQ_INIT(q); 444 TAILQ_FOREACH(m, qp, m_pqe) { 445 TAILQ_INSERT_TAIL(q, m, m_qe); 446 m->m_lock.m_owner = TID(curthread) | bit; 447 } 448 } 449 450 void 451 _mutex_fork(struct pthread *curthread) 452 { 453 454 queue_fork(curthread, &curthread->mq[TMQ_NORM], 455 &curthread->mq[TMQ_NORM_PRIV], 0); 456 queue_fork(curthread, &curthread->mq[TMQ_NORM_PP], 457 &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED); 458 queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP], 459 &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED); 460 curthread->robust_list = 0; 461 } 462 463 int 464 _thr_mutex_destroy(pthread_mutex_t *mutex) 465 { 466 pthread_mutex_t m, m1; 467 int ret; 468 469 m = *mutex; 470 if (m < THR_MUTEX_DESTROYED) { 471 ret = 0; 472 } else if (m == THR_MUTEX_DESTROYED) { 473 ret = EINVAL; 474 } else { 475 if (m == THR_PSHARED_PTR) { 476 m1 = __thr_pshared_offpage(mutex, 0); 477 if (m1 != NULL) { 478 if ((uint32_t)m1->m_lock.m_owner != 479 UMUTEX_RB_OWNERDEAD) { 480 mutex_assert_not_owned( 481 _get_curthread(), m1); 482 } 483 __thr_pshared_destroy(mutex); 484 } 485 *mutex = THR_MUTEX_DESTROYED; 486 return (0); 487 } 488 if (PMUTEX_OWNER_ID(m) != 0 && 489 (uint32_t)m->m_lock.m_owner != UMUTEX_RB_NOTRECOV) { 490 ret = EBUSY; 491 } else { 492 *mutex = THR_MUTEX_DESTROYED; 493 mutex_assert_not_owned(_get_curthread(), m); 494 __thr_free(m); 495 ret = 0; 496 } 497 } 498 499 return (ret); 500 } 501 502 static int 503 mutex_qidx(struct pthread_mutex *m) 504 { 505 506 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 507 return (TMQ_NORM); 508 return (is_robust_mutex(m) ? TMQ_ROBUST_PP : TMQ_NORM_PP); 509 } 510 511 /* 512 * Both enqueue_mutex() and dequeue_mutex() operate on the 513 * thread-private linkage of the locked mutexes and on the robust 514 * linkage. 515 * 516 * Robust list, as seen by kernel, must be consistent even in the case 517 * of thread termination at arbitrary moment. Since either enqueue or 518 * dequeue for list walked by kernel consists of rewriting a single 519 * forward pointer, it is safe. On the other hand, rewrite of the 520 * back pointer is not atomic WRT the forward one, but kernel does not 521 * care. 522 */ 523 static void 524 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m, 525 int error) 526 { 527 struct pthread_mutex *m1; 528 uintptr_t *rl; 529 int qidx; 530 531 /* Add to the list of owned mutexes: */ 532 if (error != EOWNERDEAD) 533 mutex_assert_not_owned(curthread, m); 534 qidx = mutex_qidx(m); 535 TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe); 536 if (!is_pshared_mutex(m)) 537 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe); 538 if (is_robust_mutex(m)) { 539 rl = is_pshared_mutex(m) ? &curthread->robust_list : 540 &curthread->priv_robust_list; 541 m->m_rb_prev = NULL; 542 if (*rl != 0) { 543 m1 = __containerof((void *)*rl, 544 struct pthread_mutex, m_lock); 545 m->m_lock.m_rb_lnk = (uintptr_t)&m1->m_lock; 546 m1->m_rb_prev = m; 547 } else { 548 m1 = NULL; 549 m->m_lock.m_rb_lnk = 0; 550 } 551 *rl = (uintptr_t)&m->m_lock; 552 } 553 } 554 555 static void 556 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m) 557 { 558 struct pthread_mutex *mp, *mn; 559 int qidx; 560 561 mutex_assert_is_owned(m); 562 qidx = mutex_qidx(m); 563 if (is_robust_mutex(m)) { 564 mp = m->m_rb_prev; 565 if (mp == NULL) { 566 if (is_pshared_mutex(m)) { 567 curthread->robust_list = m->m_lock.m_rb_lnk; 568 } else { 569 curthread->priv_robust_list = 570 m->m_lock.m_rb_lnk; 571 } 572 } else { 573 mp->m_lock.m_rb_lnk = m->m_lock.m_rb_lnk; 574 } 575 if (m->m_lock.m_rb_lnk != 0) { 576 mn = __containerof((void *)m->m_lock.m_rb_lnk, 577 struct pthread_mutex, m_lock); 578 mn->m_rb_prev = m->m_rb_prev; 579 } 580 m->m_lock.m_rb_lnk = 0; 581 m->m_rb_prev = NULL; 582 } 583 TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe); 584 if (!is_pshared_mutex(m)) 585 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe); 586 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0) 587 set_inherited_priority(curthread, m); 588 mutex_init_link(m); 589 } 590 591 static int 592 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m) 593 { 594 int ret; 595 596 *m = *mutex; 597 ret = 0; 598 if (__predict_false(*m == THR_PSHARED_PTR)) { 599 *m = __thr_pshared_offpage(mutex, 0); 600 if (*m == NULL) 601 ret = EINVAL; 602 else 603 shared_mutex_init(*m, NULL); 604 } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) { 605 if (*m == THR_MUTEX_DESTROYED) { 606 ret = EINVAL; 607 } else { 608 ret = init_static(_get_curthread(), mutex); 609 if (ret == 0) 610 *m = *mutex; 611 } 612 } 613 return (ret); 614 } 615 616 int 617 __Tthr_mutex_trylock(pthread_mutex_t *mutex) 618 { 619 struct pthread *curthread; 620 struct pthread_mutex *m; 621 uint32_t id; 622 int ret, robust; 623 624 _thr_check_init(); 625 ret = check_and_init_mutex(mutex, &m); 626 if (ret != 0) 627 return (ret); 628 curthread = _get_curthread(); 629 id = TID(curthread); 630 if (m->m_flags & PMUTEX_FLAG_PRIVATE) 631 THR_CRITICAL_ENTER(curthread); 632 robust = _mutex_enter_robust(curthread, m); 633 ret = _thr_umutex_trylock(&m->m_lock, id); 634 if (__predict_true(ret == 0) || ret == EOWNERDEAD) { 635 enqueue_mutex(curthread, m, ret); 636 if (ret == EOWNERDEAD) 637 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; 638 } else if (PMUTEX_OWNER_ID(m) == id) { 639 ret = mutex_self_trylock(m); 640 } /* else {} */ 641 if (robust) 642 _mutex_leave_robust(curthread, m); 643 if (ret != 0 && ret != EOWNERDEAD && 644 (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0) 645 THR_CRITICAL_LEAVE(curthread); 646 return (ret); 647 } 648 649 static int 650 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, 651 const struct timespec *abstime) 652 { 653 uint32_t id, owner; 654 int count, ret; 655 656 id = TID(curthread); 657 if (PMUTEX_OWNER_ID(m) == id) 658 return (mutex_self_lock(m, abstime)); 659 660 /* 661 * For adaptive mutexes, spin for a bit in the expectation 662 * that if the application requests this mutex type then 663 * the lock is likely to be released quickly and it is 664 * faster than entering the kernel 665 */ 666 if (__predict_false((m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT | 667 UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 0)) 668 goto sleep_in_kernel; 669 670 if (!_thr_is_smp) 671 goto yield_loop; 672 673 count = m->m_spinloops; 674 while (count--) { 675 owner = m->m_lock.m_owner; 676 if ((owner & ~UMUTEX_CONTESTED) == 0) { 677 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, 678 id | owner)) { 679 ret = 0; 680 goto done; 681 } 682 } 683 CPU_SPINWAIT; 684 } 685 686 yield_loop: 687 count = m->m_yieldloops; 688 while (count--) { 689 _sched_yield(); 690 owner = m->m_lock.m_owner; 691 if ((owner & ~UMUTEX_CONTESTED) == 0) { 692 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, 693 id | owner)) { 694 ret = 0; 695 goto done; 696 } 697 } 698 } 699 700 sleep_in_kernel: 701 if (abstime == NULL) 702 ret = __thr_umutex_lock(&m->m_lock, id); 703 else if (__predict_false(abstime->tv_nsec < 0 || 704 abstime->tv_nsec >= 1000000000)) 705 ret = EINVAL; 706 else 707 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime); 708 done: 709 if (ret == 0 || ret == EOWNERDEAD) { 710 enqueue_mutex(curthread, m, ret); 711 if (ret == EOWNERDEAD) 712 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; 713 } 714 return (ret); 715 } 716 717 static __always_inline int 718 mutex_lock_common(struct pthread_mutex *m, const struct timespec *abstime, 719 bool cvattach, bool rb_onlist) 720 { 721 struct pthread *curthread; 722 int ret, robust; 723 724 robust = 0; /* pacify gcc */ 725 curthread = _get_curthread(); 726 if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE) 727 THR_CRITICAL_ENTER(curthread); 728 if (!rb_onlist) 729 robust = _mutex_enter_robust(curthread, m); 730 ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread)); 731 if (__predict_true(ret == 0) || ret == EOWNERDEAD) { 732 enqueue_mutex(curthread, m, ret); 733 if (ret == EOWNERDEAD) 734 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; 735 } else { 736 ret = mutex_lock_sleep(curthread, m, abstime); 737 } 738 if (!rb_onlist && robust) 739 _mutex_leave_robust(curthread, m); 740 if (ret != 0 && ret != EOWNERDEAD && 741 (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0 && !cvattach) 742 THR_CRITICAL_LEAVE(curthread); 743 return (ret); 744 } 745 746 int 747 __Tthr_mutex_lock(pthread_mutex_t *mutex) 748 { 749 struct pthread_mutex *m; 750 int ret; 751 752 _thr_check_init(); 753 ret = check_and_init_mutex(mutex, &m); 754 if (ret == 0) 755 ret = mutex_lock_common(m, NULL, false, false); 756 return (ret); 757 } 758 759 int 760 __pthread_mutex_timedlock(pthread_mutex_t * __restrict mutex, 761 const struct timespec * __restrict abstime) 762 { 763 struct pthread_mutex *m; 764 int ret; 765 766 _thr_check_init(); 767 ret = check_and_init_mutex(mutex, &m); 768 if (ret == 0) 769 ret = mutex_lock_common(m, abstime, false, false); 770 return (ret); 771 } 772 773 int 774 _thr_mutex_unlock(pthread_mutex_t *mutex) 775 { 776 struct pthread_mutex *mp; 777 778 if (*mutex == THR_PSHARED_PTR) { 779 mp = __thr_pshared_offpage(mutex, 0); 780 if (mp == NULL) 781 return (EINVAL); 782 shared_mutex_init(mp, NULL); 783 } else { 784 mp = *mutex; 785 } 786 return (mutex_unlock_common(mp, false, NULL)); 787 } 788 789 int 790 _mutex_cv_lock(struct pthread_mutex *m, int count, bool rb_onlist) 791 { 792 int error; 793 794 error = mutex_lock_common(m, NULL, true, rb_onlist); 795 if (error == 0 || error == EOWNERDEAD) 796 m->m_count = count; 797 return (error); 798 } 799 800 int 801 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer) 802 { 803 804 /* 805 * Clear the count in case this is a recursive mutex. 806 */ 807 *count = m->m_count; 808 m->m_count = 0; 809 (void)mutex_unlock_common(m, true, defer); 810 return (0); 811 } 812 813 int 814 _mutex_cv_attach(struct pthread_mutex *m, int count) 815 { 816 struct pthread *curthread; 817 818 curthread = _get_curthread(); 819 enqueue_mutex(curthread, m, 0); 820 m->m_count = count; 821 return (0); 822 } 823 824 int 825 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) 826 { 827 struct pthread *curthread; 828 int deferred, error; 829 830 curthread = _get_curthread(); 831 if ((error = _mutex_owned(curthread, mp)) != 0) 832 return (error); 833 834 /* 835 * Clear the count in case this is a recursive mutex. 836 */ 837 *recurse = mp->m_count; 838 mp->m_count = 0; 839 dequeue_mutex(curthread, mp); 840 841 /* Will this happen in real-world ? */ 842 if ((mp->m_flags & PMUTEX_FLAG_DEFERRED) != 0) { 843 deferred = 1; 844 mp->m_flags &= ~PMUTEX_FLAG_DEFERRED; 845 } else 846 deferred = 0; 847 848 if (deferred) { 849 _thr_wake_all(curthread->defer_waiters, 850 curthread->nwaiter_defer); 851 curthread->nwaiter_defer = 0; 852 } 853 return (0); 854 } 855 856 static int 857 mutex_self_trylock(struct pthread_mutex *m) 858 { 859 int ret; 860 861 switch (PMUTEX_TYPE(m->m_flags)) { 862 case PTHREAD_MUTEX_ERRORCHECK: 863 case PTHREAD_MUTEX_NORMAL: 864 case PTHREAD_MUTEX_ADAPTIVE_NP: 865 ret = EBUSY; 866 break; 867 868 case PTHREAD_MUTEX_RECURSIVE: 869 /* Increment the lock count: */ 870 if (m->m_count < INT_MAX) { 871 m->m_count++; 872 ret = 0; 873 } else 874 ret = EAGAIN; 875 break; 876 877 default: 878 /* Trap invalid mutex types; */ 879 ret = EINVAL; 880 } 881 882 return (ret); 883 } 884 885 static int 886 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime) 887 { 888 struct timespec ts1, ts2; 889 int ret; 890 891 switch (PMUTEX_TYPE(m->m_flags)) { 892 case PTHREAD_MUTEX_ERRORCHECK: 893 case PTHREAD_MUTEX_ADAPTIVE_NP: 894 if (abstime) { 895 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 896 abstime->tv_nsec >= 1000000000) { 897 ret = EINVAL; 898 } else { 899 clock_gettime(CLOCK_REALTIME, &ts1); 900 TIMESPEC_SUB(&ts2, abstime, &ts1); 901 __sys_nanosleep(&ts2, NULL); 902 ret = ETIMEDOUT; 903 } 904 } else { 905 /* 906 * POSIX specifies that mutexes should return 907 * EDEADLK if a recursive lock is detected. 908 */ 909 ret = EDEADLK; 910 } 911 break; 912 913 case PTHREAD_MUTEX_NORMAL: 914 /* 915 * What SS2 define as a 'normal' mutex. Intentionally 916 * deadlock on attempts to get a lock you already own. 917 */ 918 ret = 0; 919 if (abstime) { 920 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 921 abstime->tv_nsec >= 1000000000) { 922 ret = EINVAL; 923 } else { 924 clock_gettime(CLOCK_REALTIME, &ts1); 925 TIMESPEC_SUB(&ts2, abstime, &ts1); 926 __sys_nanosleep(&ts2, NULL); 927 ret = ETIMEDOUT; 928 } 929 } else { 930 ts1.tv_sec = 30; 931 ts1.tv_nsec = 0; 932 for (;;) 933 __sys_nanosleep(&ts1, NULL); 934 } 935 break; 936 937 case PTHREAD_MUTEX_RECURSIVE: 938 /* Increment the lock count: */ 939 if (m->m_count < INT_MAX) { 940 m->m_count++; 941 ret = 0; 942 } else 943 ret = EAGAIN; 944 break; 945 946 default: 947 /* Trap invalid mutex types; */ 948 ret = EINVAL; 949 } 950 951 return (ret); 952 } 953 954 static __always_inline int 955 mutex_unlock_common(struct pthread_mutex *m, bool cv, int *mtx_defer) 956 { 957 struct pthread *curthread; 958 uint32_t id; 959 int deferred, error, private, robust; 960 961 if (__predict_false(m <= THR_MUTEX_DESTROYED)) { 962 if (m == THR_MUTEX_DESTROYED) 963 return (EINVAL); 964 return (EPERM); 965 } 966 967 curthread = _get_curthread(); 968 id = TID(curthread); 969 970 /* 971 * Check if the running thread is not the owner of the mutex. 972 */ 973 if (__predict_false(PMUTEX_OWNER_ID(m) != id)) 974 return (EPERM); 975 976 error = 0; 977 private = (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0; 978 if (__predict_false(PMUTEX_TYPE(m->m_flags) == 979 PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) { 980 m->m_count--; 981 } else { 982 if ((m->m_flags & PMUTEX_FLAG_DEFERRED) != 0) { 983 deferred = 1; 984 m->m_flags &= ~PMUTEX_FLAG_DEFERRED; 985 } else 986 deferred = 0; 987 988 robust = _mutex_enter_robust(curthread, m); 989 dequeue_mutex(curthread, m); 990 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer); 991 if (deferred) { 992 if (mtx_defer == NULL) { 993 _thr_wake_all(curthread->defer_waiters, 994 curthread->nwaiter_defer); 995 curthread->nwaiter_defer = 0; 996 } else 997 *mtx_defer = 1; 998 } 999 if (robust) 1000 _mutex_leave_robust(curthread, m); 1001 } 1002 if (!cv && private) 1003 THR_CRITICAL_LEAVE(curthread); 1004 return (error); 1005 } 1006 1007 int 1008 _pthread_mutex_getprioceiling(const pthread_mutex_t * __restrict mutex, 1009 int * __restrict prioceiling) 1010 { 1011 struct pthread_mutex *m; 1012 1013 if (*mutex == THR_PSHARED_PTR) { 1014 m = __thr_pshared_offpage(__DECONST(void *, mutex), 0); 1015 if (m == NULL) 1016 return (EINVAL); 1017 shared_mutex_init(m, NULL); 1018 } else { 1019 m = *mutex; 1020 if (m <= THR_MUTEX_DESTROYED) 1021 return (EINVAL); 1022 } 1023 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 1024 return (EINVAL); 1025 *prioceiling = m->m_lock.m_ceilings[0]; 1026 return (0); 1027 } 1028 1029 int 1030 _pthread_mutex_setprioceiling(pthread_mutex_t * __restrict mutex, 1031 int ceiling, int * __restrict old_ceiling) 1032 { 1033 struct pthread *curthread; 1034 struct pthread_mutex *m, *m1, *m2; 1035 struct mutex_queue *q, *qp; 1036 int qidx, ret; 1037 1038 if (*mutex == THR_PSHARED_PTR) { 1039 m = __thr_pshared_offpage(mutex, 0); 1040 if (m == NULL) 1041 return (EINVAL); 1042 shared_mutex_init(m, NULL); 1043 } else { 1044 m = *mutex; 1045 if (m <= THR_MUTEX_DESTROYED) 1046 return (EINVAL); 1047 } 1048 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 1049 return (EINVAL); 1050 1051 ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling); 1052 if (ret != 0) 1053 return (ret); 1054 1055 curthread = _get_curthread(); 1056 if (PMUTEX_OWNER_ID(m) == TID(curthread)) { 1057 mutex_assert_is_owned(m); 1058 m1 = TAILQ_PREV(m, mutex_queue, m_qe); 1059 m2 = TAILQ_NEXT(m, m_qe); 1060 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) || 1061 (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) { 1062 qidx = mutex_qidx(m); 1063 q = &curthread->mq[qidx]; 1064 qp = &curthread->mq[qidx + 1]; 1065 TAILQ_REMOVE(q, m, m_qe); 1066 if (!is_pshared_mutex(m)) 1067 TAILQ_REMOVE(qp, m, m_pqe); 1068 TAILQ_FOREACH(m2, q, m_qe) { 1069 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) { 1070 TAILQ_INSERT_BEFORE(m2, m, m_qe); 1071 if (!is_pshared_mutex(m)) { 1072 while (m2 != NULL && 1073 is_pshared_mutex(m2)) { 1074 m2 = TAILQ_PREV(m2, 1075 mutex_queue, m_qe); 1076 } 1077 if (m2 == NULL) { 1078 TAILQ_INSERT_HEAD(qp, 1079 m, m_pqe); 1080 } else { 1081 TAILQ_INSERT_BEFORE(m2, 1082 m, m_pqe); 1083 } 1084 } 1085 return (0); 1086 } 1087 } 1088 TAILQ_INSERT_TAIL(q, m, m_qe); 1089 if (!is_pshared_mutex(m)) 1090 TAILQ_INSERT_TAIL(qp, m, m_pqe); 1091 } 1092 } 1093 return (0); 1094 } 1095 1096 int 1097 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count) 1098 { 1099 struct pthread_mutex *m; 1100 int ret; 1101 1102 ret = check_and_init_mutex(mutex, &m); 1103 if (ret == 0) 1104 *count = m->m_spinloops; 1105 return (ret); 1106 } 1107 1108 int 1109 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count) 1110 { 1111 struct pthread_mutex *m; 1112 int ret; 1113 1114 if (count < 0) 1115 return (EINVAL); 1116 ret = check_and_init_mutex(mutex, &m); 1117 if (ret == 0) 1118 m->m_spinloops = count; 1119 return (ret); 1120 } 1121 1122 int 1123 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count) 1124 { 1125 struct pthread_mutex *m; 1126 int ret; 1127 1128 ret = check_and_init_mutex(mutex, &m); 1129 if (ret == 0) 1130 *count = m->m_yieldloops; 1131 return (ret); 1132 } 1133 1134 int 1135 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count) 1136 { 1137 struct pthread_mutex *m; 1138 int ret; 1139 1140 if (count < 0) 1141 return (EINVAL); 1142 ret = check_and_init_mutex(mutex, &m); 1143 if (ret == 0) 1144 m->m_yieldloops = count; 1145 return (ret); 1146 } 1147 1148 int 1149 _pthread_mutex_isowned_np(pthread_mutex_t *mutex) 1150 { 1151 struct pthread_mutex *m; 1152 1153 if (*mutex == THR_PSHARED_PTR) { 1154 m = __thr_pshared_offpage(mutex, 0); 1155 if (m == NULL) 1156 return (0); 1157 shared_mutex_init(m, NULL); 1158 } else { 1159 m = *mutex; 1160 if (m <= THR_MUTEX_DESTROYED) 1161 return (0); 1162 } 1163 return (PMUTEX_OWNER_ID(m) == TID(_get_curthread())); 1164 } 1165 1166 int 1167 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp) 1168 { 1169 1170 if (__predict_false(mp <= THR_MUTEX_DESTROYED)) { 1171 if (mp == THR_MUTEX_DESTROYED) 1172 return (EINVAL); 1173 return (EPERM); 1174 } 1175 if (PMUTEX_OWNER_ID(mp) != TID(curthread)) 1176 return (EPERM); 1177 return (0); 1178 } 1179 1180 int 1181 _Tthr_mutex_consistent(pthread_mutex_t *mutex) 1182 { 1183 struct pthread_mutex *m; 1184 struct pthread *curthread; 1185 1186 if (*mutex == THR_PSHARED_PTR) { 1187 m = __thr_pshared_offpage(mutex, 0); 1188 if (m == NULL) 1189 return (EINVAL); 1190 shared_mutex_init(m, NULL); 1191 } else { 1192 m = *mutex; 1193 if (m <= THR_MUTEX_DESTROYED) 1194 return (EINVAL); 1195 } 1196 curthread = _get_curthread(); 1197 if ((m->m_lock.m_flags & (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 1198 (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) 1199 return (EINVAL); 1200 if (PMUTEX_OWNER_ID(m) != TID(curthread)) 1201 return (EPERM); 1202 m->m_lock.m_flags &= ~UMUTEX_NONCONSISTENT; 1203 return (0); 1204 } 1205