1 /*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. 5 * Copyright (c) 2006 David Xu <davidxu@freebsd.org>. 6 * Copyright (c) 2015, 2016 The FreeBSD Foundation 7 * 8 * All rights reserved. 9 * 10 * Portions of this software were developed by Konstantin Belousov 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by John Birrell. 24 * 4. Neither the name of the author nor the names of any co-contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/cdefs.h> 42 #include "namespace.h" 43 #include <stdlib.h> 44 #include <errno.h> 45 #include <string.h> 46 #include <sys/param.h> 47 #include <sys/queue.h> 48 #include <pthread.h> 49 #include <pthread_np.h> 50 #include "un-namespace.h" 51 52 #include "thr_private.h" 53 54 _Static_assert(sizeof(struct pthread_mutex) <= THR_PAGE_SIZE_MIN, 55 "pthread_mutex is too large for off-page"); 56 57 /* 58 * For adaptive mutexes, how many times to spin doing trylock2 59 * before entering the kernel to block 60 */ 61 #define MUTEX_ADAPTIVE_SPINS 2000 62 63 /* 64 * Prototypes 65 */ 66 int __pthread_mutex_timedlock(pthread_mutex_t * __restrict mutex, 67 const struct timespec * __restrict abstime); 68 int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count); 69 int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 70 int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 71 int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 72 int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count); 73 int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 74 75 static int mutex_self_trylock(pthread_mutex_t); 76 static int mutex_self_lock(pthread_mutex_t, 77 const struct timespec *abstime); 78 static int mutex_unlock_common(struct pthread_mutex *, bool, int *); 79 static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, 80 const struct timespec *); 81 static void mutex_init_robust(struct pthread *curthread); 82 static int mutex_qidx(struct pthread_mutex *m); 83 static bool is_robust_mutex(struct pthread_mutex *m); 84 static bool is_pshared_mutex(struct pthread_mutex *m); 85 86 __weak_reference(__Tthr_mutex_init, pthread_mutex_init); 87 __weak_reference(__Tthr_mutex_init, __pthread_mutex_init); 88 __strong_reference(__Tthr_mutex_init, _pthread_mutex_init); 89 __weak_reference(__Tthr_mutex_lock, pthread_mutex_lock); 90 __weak_reference(__Tthr_mutex_lock, __pthread_mutex_lock); 91 __strong_reference(__Tthr_mutex_lock, _pthread_mutex_lock); 92 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); 93 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock); 94 __weak_reference(__Tthr_mutex_trylock, pthread_mutex_trylock); 95 __weak_reference(__Tthr_mutex_trylock, __pthread_mutex_trylock); 96 __strong_reference(__Tthr_mutex_trylock, _pthread_mutex_trylock); 97 __weak_reference(_Tthr_mutex_consistent, pthread_mutex_consistent); 98 __weak_reference(_Tthr_mutex_consistent, _pthread_mutex_consistent); 99 __strong_reference(_Tthr_mutex_consistent, __pthread_mutex_consistent); 100 101 /* Single underscore versions provided for libc internal usage: */ 102 /* No difference between libc and application usage of these: */ 103 __weak_reference(_thr_mutex_destroy, pthread_mutex_destroy); 104 __weak_reference(_thr_mutex_destroy, _pthread_mutex_destroy); 105 __weak_reference(_thr_mutex_unlock, pthread_mutex_unlock); 106 __weak_reference(_thr_mutex_unlock, _pthread_mutex_unlock); 107 108 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling); 109 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling); 110 111 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np); 112 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np); 113 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np); 114 115 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np); 116 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np); 117 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np); 118 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np); 119 120 static void 121 mutex_init_link(struct pthread_mutex *m __unused) 122 { 123 124 #if defined(_PTHREADS_INVARIANTS) 125 m->m_qe.tqe_prev = NULL; 126 m->m_qe.tqe_next = NULL; 127 m->m_pqe.tqe_prev = NULL; 128 m->m_pqe.tqe_next = NULL; 129 #endif 130 } 131 132 static void 133 mutex_assert_is_owned(struct pthread_mutex *m __unused) 134 { 135 136 #if defined(_PTHREADS_INVARIANTS) 137 if (__predict_false(m->m_qe.tqe_prev == NULL)) 138 PANIC("mutex %p own %#x is not on list %p %p", 139 m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); 140 #endif 141 } 142 143 static void 144 mutex_assert_not_owned(struct pthread *curthread __unused, 145 struct pthread_mutex *m __unused) 146 { 147 148 #if defined(_PTHREADS_INVARIANTS) 149 if (__predict_false(m->m_qe.tqe_prev != NULL || 150 m->m_qe.tqe_next != NULL)) 151 PANIC("mutex %p own %#x is on list %p %p", 152 m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); 153 if (__predict_false(is_robust_mutex(m) && 154 (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL || 155 (is_pshared_mutex(m) && curthread->robust_list == 156 (uintptr_t)&m->m_lock) || 157 (!is_pshared_mutex(m) && curthread->priv_robust_list == 158 (uintptr_t)&m->m_lock)))) 159 PANIC( 160 "mutex %p own %#x is on robust linkage %p %p head %p phead %p", 161 m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk, 162 m->m_rb_prev, (void *)curthread->robust_list, 163 (void *)curthread->priv_robust_list); 164 #endif 165 } 166 167 static bool 168 is_pshared_mutex(struct pthread_mutex *m) 169 { 170 171 return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0); 172 } 173 174 static bool 175 is_robust_mutex(struct pthread_mutex *m) 176 { 177 178 return ((m->m_lock.m_flags & UMUTEX_ROBUST) != 0); 179 } 180 181 int 182 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m) 183 { 184 185 #if defined(_PTHREADS_INVARIANTS) 186 if (__predict_false(curthread->inact_mtx != 0)) 187 PANIC("inact_mtx enter"); 188 #endif 189 if (!is_robust_mutex(m)) 190 return (0); 191 192 mutex_init_robust(curthread); 193 curthread->inact_mtx = (uintptr_t)&m->m_lock; 194 return (1); 195 } 196 197 void 198 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused) 199 { 200 201 #if defined(_PTHREADS_INVARIANTS) 202 if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock)) 203 PANIC("inact_mtx leave"); 204 #endif 205 curthread->inact_mtx = 0; 206 } 207 208 static int 209 mutex_check_attr(const struct pthread_mutex_attr *attr) 210 { 211 212 if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK || 213 attr->m_type >= PTHREAD_MUTEX_TYPE_MAX) 214 return (EINVAL); 215 if (attr->m_protocol < PTHREAD_PRIO_NONE || 216 attr->m_protocol > PTHREAD_PRIO_PROTECT) 217 return (EINVAL); 218 return (0); 219 } 220 221 static void 222 mutex_init_robust(struct pthread *curthread) 223 { 224 struct umtx_robust_lists_params rb; 225 226 if (curthread == NULL) 227 curthread = _get_curthread(); 228 if (curthread->robust_inited) 229 return; 230 rb.robust_list_offset = (uintptr_t)&curthread->robust_list; 231 rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list; 232 rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx; 233 _umtx_op(NULL, UMTX_OP_ROBUST_LISTS, sizeof(rb), &rb, NULL); 234 curthread->robust_inited = 1; 235 } 236 237 static void 238 mutex_init_body(struct pthread_mutex *pmutex, 239 const struct pthread_mutex_attr *attr) 240 { 241 242 pmutex->m_flags = attr->m_type; 243 pmutex->m_count = 0; 244 pmutex->m_spinloops = 0; 245 pmutex->m_yieldloops = 0; 246 mutex_init_link(pmutex); 247 switch (attr->m_protocol) { 248 case PTHREAD_PRIO_NONE: 249 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 250 pmutex->m_lock.m_flags = 0; 251 break; 252 case PTHREAD_PRIO_INHERIT: 253 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 254 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT; 255 break; 256 case PTHREAD_PRIO_PROTECT: 257 pmutex->m_lock.m_owner = UMUTEX_CONTESTED; 258 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT; 259 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling; 260 break; 261 } 262 if (attr->m_pshared == PTHREAD_PROCESS_SHARED) 263 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED; 264 if (attr->m_robust == PTHREAD_MUTEX_ROBUST) { 265 mutex_init_robust(NULL); 266 pmutex->m_lock.m_flags |= UMUTEX_ROBUST; 267 } 268 if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) { 269 pmutex->m_spinloops = 270 _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS; 271 pmutex->m_yieldloops = _thr_yieldloops; 272 } 273 } 274 275 static int 276 mutex_init(pthread_mutex_t *mutex, 277 const struct pthread_mutex_attr *mutex_attr, 278 void *(calloc_cb)(size_t, size_t)) 279 { 280 const struct pthread_mutex_attr *attr; 281 struct pthread_mutex *pmutex; 282 int error; 283 284 if (mutex_attr == NULL) { 285 attr = &_pthread_mutexattr_default; 286 } else { 287 attr = mutex_attr; 288 error = mutex_check_attr(attr); 289 if (error != 0) 290 return (error); 291 } 292 if ((pmutex = (pthread_mutex_t)calloc_cb(1, 293 sizeof(struct pthread_mutex))) == NULL) 294 return (ENOMEM); 295 mutex_init_body(pmutex, attr); 296 *mutex = pmutex; 297 return (0); 298 } 299 300 static int 301 init_static(struct pthread *thread, pthread_mutex_t *mutex) 302 { 303 int ret; 304 305 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 306 307 if (*mutex == THR_MUTEX_INITIALIZER) 308 ret = mutex_init(mutex, &_pthread_mutexattr_default, 309 __thr_calloc); 310 else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) 311 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, 312 __thr_calloc); 313 else 314 ret = 0; 315 THR_LOCK_RELEASE(thread, &_mutex_static_lock); 316 317 return (ret); 318 } 319 320 static void 321 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) 322 { 323 struct pthread_mutex *m2; 324 325 m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue); 326 if (m2 != NULL) 327 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0]; 328 else 329 m->m_lock.m_ceilings[1] = -1; 330 } 331 332 static void 333 shared_mutex_init(struct pthread_mutex *pmtx, const struct 334 pthread_mutex_attr *mutex_attr) 335 { 336 static const struct pthread_mutex_attr foobar_mutex_attr = { 337 .m_type = PTHREAD_MUTEX_DEFAULT, 338 .m_protocol = PTHREAD_PRIO_NONE, 339 .m_ceiling = 0, 340 .m_pshared = PTHREAD_PROCESS_SHARED, 341 .m_robust = PTHREAD_MUTEX_STALLED, 342 }; 343 bool done; 344 345 /* 346 * Hack to allow multiple pthread_mutex_init() calls on the 347 * same process-shared mutex. We rely on kernel allocating 348 * zeroed offpage for the mutex, i.e. the 349 * PMUTEX_INITSTAGE_ALLOC value must be zero. 350 */ 351 for (done = false; !done;) { 352 switch (pmtx->m_ps) { 353 case PMUTEX_INITSTAGE_DONE: 354 atomic_thread_fence_acq(); 355 done = true; 356 break; 357 case PMUTEX_INITSTAGE_ALLOC: 358 if (atomic_cmpset_int(&pmtx->m_ps, 359 PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) { 360 if (mutex_attr == NULL) 361 mutex_attr = &foobar_mutex_attr; 362 mutex_init_body(pmtx, mutex_attr); 363 atomic_store_rel_int(&pmtx->m_ps, 364 PMUTEX_INITSTAGE_DONE); 365 done = true; 366 } 367 break; 368 case PMUTEX_INITSTAGE_BUSY: 369 _pthread_yield(); 370 break; 371 default: 372 PANIC("corrupted offpage"); 373 break; 374 } 375 } 376 } 377 378 int 379 __Tthr_mutex_init(pthread_mutex_t * __restrict mutex, 380 const pthread_mutexattr_t * __restrict mutex_attr) 381 { 382 struct pthread_mutex *pmtx; 383 int ret; 384 385 _thr_check_init(); 386 387 if (mutex_attr != NULL) { 388 ret = mutex_check_attr(*mutex_attr); 389 if (ret != 0) 390 return (ret); 391 } 392 if (mutex_attr == NULL || 393 (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { 394 __thr_malloc_init(); 395 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, 396 __thr_calloc)); 397 } 398 pmtx = __thr_pshared_offpage(__DECONST(void *, mutex), 1); 399 if (pmtx == NULL) 400 return (EFAULT); 401 *mutex = THR_PSHARED_PTR; 402 shared_mutex_init(pmtx, *mutex_attr); 403 return (0); 404 } 405 406 /* This function is used internally by malloc. */ 407 int 408 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 409 void *(calloc_cb)(size_t, size_t)) 410 { 411 static const struct pthread_mutex_attr attr = { 412 .m_type = PTHREAD_MUTEX_NORMAL, 413 .m_protocol = PTHREAD_PRIO_NONE, 414 .m_ceiling = 0, 415 .m_pshared = PTHREAD_PROCESS_PRIVATE, 416 .m_robust = PTHREAD_MUTEX_STALLED, 417 }; 418 int ret; 419 420 ret = mutex_init(mutex, &attr, calloc_cb); 421 if (ret == 0) 422 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE; 423 return (ret); 424 } 425 426 /* 427 * Fix mutex ownership for child process. 428 * 429 * Process private mutex ownership is transmitted from the forking 430 * thread to the child process. 431 * 432 * Process shared mutex should not be inherited because owner is 433 * forking thread which is in parent process, they are removed from 434 * the owned mutex list. 435 */ 436 static void 437 queue_fork(struct pthread *curthread, struct mutex_queue *q, 438 struct mutex_queue *qp, uint bit) 439 { 440 struct pthread_mutex *m; 441 442 TAILQ_INIT(q); 443 TAILQ_FOREACH(m, qp, m_pqe) { 444 TAILQ_INSERT_TAIL(q, m, m_qe); 445 m->m_lock.m_owner = TID(curthread) | bit; 446 } 447 } 448 449 void 450 _mutex_fork(struct pthread *curthread) 451 { 452 453 queue_fork(curthread, &curthread->mq[TMQ_NORM], 454 &curthread->mq[TMQ_NORM_PRIV], 0); 455 queue_fork(curthread, &curthread->mq[TMQ_NORM_PP], 456 &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED); 457 queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP], 458 &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED); 459 curthread->robust_list = 0; 460 } 461 462 int 463 _thr_mutex_destroy(pthread_mutex_t *mutex) 464 { 465 pthread_mutex_t m, m1; 466 int ret; 467 468 m = *mutex; 469 if (m < THR_MUTEX_DESTROYED) { 470 ret = 0; 471 } else if (m == THR_MUTEX_DESTROYED) { 472 ret = EINVAL; 473 } else { 474 if (m == THR_PSHARED_PTR) { 475 m1 = __thr_pshared_offpage(mutex, 0); 476 if (m1 != NULL) { 477 if ((uint32_t)m1->m_lock.m_owner != 478 UMUTEX_RB_OWNERDEAD) { 479 mutex_assert_not_owned( 480 _get_curthread(), m1); 481 } 482 __thr_pshared_destroy(mutex); 483 } 484 *mutex = THR_MUTEX_DESTROYED; 485 return (0); 486 } 487 if (PMUTEX_OWNER_ID(m) != 0 && 488 (uint32_t)m->m_lock.m_owner != UMUTEX_RB_NOTRECOV) { 489 ret = EBUSY; 490 } else { 491 *mutex = THR_MUTEX_DESTROYED; 492 mutex_assert_not_owned(_get_curthread(), m); 493 __thr_free(m); 494 ret = 0; 495 } 496 } 497 498 return (ret); 499 } 500 501 static int 502 mutex_qidx(struct pthread_mutex *m) 503 { 504 505 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 506 return (TMQ_NORM); 507 return (is_robust_mutex(m) ? TMQ_ROBUST_PP : TMQ_NORM_PP); 508 } 509 510 /* 511 * Both enqueue_mutex() and dequeue_mutex() operate on the 512 * thread-private linkage of the locked mutexes and on the robust 513 * linkage. 514 * 515 * Robust list, as seen by kernel, must be consistent even in the case 516 * of thread termination at arbitrary moment. Since either enqueue or 517 * dequeue for list walked by kernel consists of rewriting a single 518 * forward pointer, it is safe. On the other hand, rewrite of the 519 * back pointer is not atomic WRT the forward one, but kernel does not 520 * care. 521 */ 522 static void 523 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m, 524 int error) 525 { 526 struct pthread_mutex *m1; 527 uintptr_t *rl; 528 int qidx; 529 530 /* Add to the list of owned mutexes: */ 531 if (error != EOWNERDEAD) 532 mutex_assert_not_owned(curthread, m); 533 qidx = mutex_qidx(m); 534 TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe); 535 if (!is_pshared_mutex(m)) 536 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe); 537 if (is_robust_mutex(m)) { 538 rl = is_pshared_mutex(m) ? &curthread->robust_list : 539 &curthread->priv_robust_list; 540 m->m_rb_prev = NULL; 541 if (*rl != 0) { 542 m1 = __containerof((void *)*rl, 543 struct pthread_mutex, m_lock); 544 m->m_lock.m_rb_lnk = (uintptr_t)&m1->m_lock; 545 m1->m_rb_prev = m; 546 } else { 547 m1 = NULL; 548 m->m_lock.m_rb_lnk = 0; 549 } 550 *rl = (uintptr_t)&m->m_lock; 551 } 552 } 553 554 static void 555 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m) 556 { 557 struct pthread_mutex *mp, *mn; 558 int qidx; 559 560 mutex_assert_is_owned(m); 561 qidx = mutex_qidx(m); 562 if (is_robust_mutex(m)) { 563 mp = m->m_rb_prev; 564 if (mp == NULL) { 565 if (is_pshared_mutex(m)) { 566 curthread->robust_list = m->m_lock.m_rb_lnk; 567 } else { 568 curthread->priv_robust_list = 569 m->m_lock.m_rb_lnk; 570 } 571 } else { 572 mp->m_lock.m_rb_lnk = m->m_lock.m_rb_lnk; 573 } 574 if (m->m_lock.m_rb_lnk != 0) { 575 mn = __containerof((void *)m->m_lock.m_rb_lnk, 576 struct pthread_mutex, m_lock); 577 mn->m_rb_prev = m->m_rb_prev; 578 } 579 m->m_lock.m_rb_lnk = 0; 580 m->m_rb_prev = NULL; 581 } 582 TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe); 583 if (!is_pshared_mutex(m)) 584 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe); 585 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0) 586 set_inherited_priority(curthread, m); 587 mutex_init_link(m); 588 } 589 590 static int 591 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m) 592 { 593 int ret; 594 595 *m = *mutex; 596 ret = 0; 597 if (__predict_false(*m == THR_PSHARED_PTR)) { 598 *m = __thr_pshared_offpage(mutex, 0); 599 if (*m == NULL) 600 ret = EINVAL; 601 else 602 shared_mutex_init(*m, NULL); 603 } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) { 604 if (*m == THR_MUTEX_DESTROYED) { 605 ret = EINVAL; 606 } else { 607 ret = init_static(_get_curthread(), mutex); 608 if (ret == 0) 609 *m = *mutex; 610 } 611 } 612 return (ret); 613 } 614 615 int 616 __Tthr_mutex_trylock(pthread_mutex_t *mutex) 617 { 618 struct pthread *curthread; 619 struct pthread_mutex *m; 620 uint32_t id; 621 int ret, robust; 622 623 ret = check_and_init_mutex(mutex, &m); 624 if (ret != 0) 625 return (ret); 626 curthread = _get_curthread(); 627 id = TID(curthread); 628 if (m->m_flags & PMUTEX_FLAG_PRIVATE) 629 THR_CRITICAL_ENTER(curthread); 630 robust = _mutex_enter_robust(curthread, m); 631 ret = _thr_umutex_trylock(&m->m_lock, id); 632 if (__predict_true(ret == 0) || ret == EOWNERDEAD) { 633 enqueue_mutex(curthread, m, ret); 634 if (ret == EOWNERDEAD) 635 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; 636 } else if (PMUTEX_OWNER_ID(m) == id) { 637 ret = mutex_self_trylock(m); 638 } /* else {} */ 639 if (robust) 640 _mutex_leave_robust(curthread, m); 641 if (ret != 0 && ret != EOWNERDEAD && 642 (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0) 643 THR_CRITICAL_LEAVE(curthread); 644 return (ret); 645 } 646 647 static int 648 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, 649 const struct timespec *abstime) 650 { 651 uint32_t id, owner; 652 int count, ret; 653 654 id = TID(curthread); 655 if (PMUTEX_OWNER_ID(m) == id) 656 return (mutex_self_lock(m, abstime)); 657 658 /* 659 * For adaptive mutexes, spin for a bit in the expectation 660 * that if the application requests this mutex type then 661 * the lock is likely to be released quickly and it is 662 * faster than entering the kernel 663 */ 664 if (__predict_false((m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT | 665 UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 0)) 666 goto sleep_in_kernel; 667 668 if (!_thr_is_smp) 669 goto yield_loop; 670 671 count = m->m_spinloops; 672 while (count--) { 673 owner = m->m_lock.m_owner; 674 if ((owner & ~UMUTEX_CONTESTED) == 0) { 675 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, 676 id | owner)) { 677 ret = 0; 678 goto done; 679 } 680 } 681 CPU_SPINWAIT; 682 } 683 684 yield_loop: 685 count = m->m_yieldloops; 686 while (count--) { 687 _sched_yield(); 688 owner = m->m_lock.m_owner; 689 if ((owner & ~UMUTEX_CONTESTED) == 0) { 690 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, 691 id | owner)) { 692 ret = 0; 693 goto done; 694 } 695 } 696 } 697 698 sleep_in_kernel: 699 if (abstime == NULL) 700 ret = __thr_umutex_lock(&m->m_lock, id); 701 else if (__predict_false(abstime->tv_nsec < 0 || 702 abstime->tv_nsec >= 1000000000)) 703 ret = EINVAL; 704 else 705 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime); 706 done: 707 if (ret == 0 || ret == EOWNERDEAD) { 708 enqueue_mutex(curthread, m, ret); 709 if (ret == EOWNERDEAD) 710 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; 711 } 712 return (ret); 713 } 714 715 static __always_inline int 716 mutex_lock_common(struct pthread_mutex *m, const struct timespec *abstime, 717 bool cvattach, bool rb_onlist) 718 { 719 struct pthread *curthread; 720 int ret, robust; 721 722 robust = 0; /* pacify gcc */ 723 curthread = _get_curthread(); 724 if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE) 725 THR_CRITICAL_ENTER(curthread); 726 if (!rb_onlist) 727 robust = _mutex_enter_robust(curthread, m); 728 ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread)); 729 if (__predict_true(ret == 0) || ret == EOWNERDEAD) { 730 enqueue_mutex(curthread, m, ret); 731 if (ret == EOWNERDEAD) 732 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; 733 } else { 734 ret = mutex_lock_sleep(curthread, m, abstime); 735 } 736 if (!rb_onlist && robust) 737 _mutex_leave_robust(curthread, m); 738 if (ret != 0 && ret != EOWNERDEAD && 739 (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0 && !cvattach) 740 THR_CRITICAL_LEAVE(curthread); 741 return (ret); 742 } 743 744 int 745 __Tthr_mutex_lock(pthread_mutex_t *mutex) 746 { 747 struct pthread_mutex *m; 748 int ret; 749 750 _thr_check_init(); 751 ret = check_and_init_mutex(mutex, &m); 752 if (ret == 0) 753 ret = mutex_lock_common(m, NULL, false, false); 754 return (ret); 755 } 756 757 int 758 __pthread_mutex_timedlock(pthread_mutex_t * __restrict mutex, 759 const struct timespec * __restrict abstime) 760 { 761 struct pthread_mutex *m; 762 int ret; 763 764 _thr_check_init(); 765 ret = check_and_init_mutex(mutex, &m); 766 if (ret == 0) 767 ret = mutex_lock_common(m, abstime, false, false); 768 return (ret); 769 } 770 771 int 772 _thr_mutex_unlock(pthread_mutex_t *mutex) 773 { 774 struct pthread_mutex *mp; 775 776 if (*mutex == THR_PSHARED_PTR) { 777 mp = __thr_pshared_offpage(mutex, 0); 778 if (mp == NULL) 779 return (EINVAL); 780 shared_mutex_init(mp, NULL); 781 } else { 782 mp = *mutex; 783 } 784 return (mutex_unlock_common(mp, false, NULL)); 785 } 786 787 int 788 _mutex_cv_lock(struct pthread_mutex *m, int count, bool rb_onlist) 789 { 790 int error; 791 792 error = mutex_lock_common(m, NULL, true, rb_onlist); 793 if (error == 0 || error == EOWNERDEAD) 794 m->m_count = count; 795 return (error); 796 } 797 798 int 799 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer) 800 { 801 802 /* 803 * Clear the count in case this is a recursive mutex. 804 */ 805 *count = m->m_count; 806 m->m_count = 0; 807 (void)mutex_unlock_common(m, true, defer); 808 return (0); 809 } 810 811 int 812 _mutex_cv_attach(struct pthread_mutex *m, int count) 813 { 814 struct pthread *curthread; 815 816 curthread = _get_curthread(); 817 enqueue_mutex(curthread, m, 0); 818 m->m_count = count; 819 return (0); 820 } 821 822 int 823 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) 824 { 825 struct pthread *curthread; 826 int deferred, error; 827 828 curthread = _get_curthread(); 829 if ((error = _mutex_owned(curthread, mp)) != 0) 830 return (error); 831 832 /* 833 * Clear the count in case this is a recursive mutex. 834 */ 835 *recurse = mp->m_count; 836 mp->m_count = 0; 837 dequeue_mutex(curthread, mp); 838 839 /* Will this happen in real-world ? */ 840 if ((mp->m_flags & PMUTEX_FLAG_DEFERRED) != 0) { 841 deferred = 1; 842 mp->m_flags &= ~PMUTEX_FLAG_DEFERRED; 843 } else 844 deferred = 0; 845 846 if (deferred) { 847 _thr_wake_all(curthread->defer_waiters, 848 curthread->nwaiter_defer); 849 curthread->nwaiter_defer = 0; 850 } 851 return (0); 852 } 853 854 static int 855 mutex_self_trylock(struct pthread_mutex *m) 856 { 857 int ret; 858 859 switch (PMUTEX_TYPE(m->m_flags)) { 860 case PTHREAD_MUTEX_ERRORCHECK: 861 case PTHREAD_MUTEX_NORMAL: 862 case PTHREAD_MUTEX_ADAPTIVE_NP: 863 ret = EBUSY; 864 break; 865 866 case PTHREAD_MUTEX_RECURSIVE: 867 /* Increment the lock count: */ 868 if (m->m_count + 1 > 0) { 869 m->m_count++; 870 ret = 0; 871 } else 872 ret = EAGAIN; 873 break; 874 875 default: 876 /* Trap invalid mutex types; */ 877 ret = EINVAL; 878 } 879 880 return (ret); 881 } 882 883 static int 884 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime) 885 { 886 struct timespec ts1, ts2; 887 int ret; 888 889 switch (PMUTEX_TYPE(m->m_flags)) { 890 case PTHREAD_MUTEX_ERRORCHECK: 891 case PTHREAD_MUTEX_ADAPTIVE_NP: 892 if (abstime) { 893 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 894 abstime->tv_nsec >= 1000000000) { 895 ret = EINVAL; 896 } else { 897 clock_gettime(CLOCK_REALTIME, &ts1); 898 TIMESPEC_SUB(&ts2, abstime, &ts1); 899 __sys_nanosleep(&ts2, NULL); 900 ret = ETIMEDOUT; 901 } 902 } else { 903 /* 904 * POSIX specifies that mutexes should return 905 * EDEADLK if a recursive lock is detected. 906 */ 907 ret = EDEADLK; 908 } 909 break; 910 911 case PTHREAD_MUTEX_NORMAL: 912 /* 913 * What SS2 define as a 'normal' mutex. Intentionally 914 * deadlock on attempts to get a lock you already own. 915 */ 916 ret = 0; 917 if (abstime) { 918 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 919 abstime->tv_nsec >= 1000000000) { 920 ret = EINVAL; 921 } else { 922 clock_gettime(CLOCK_REALTIME, &ts1); 923 TIMESPEC_SUB(&ts2, abstime, &ts1); 924 __sys_nanosleep(&ts2, NULL); 925 ret = ETIMEDOUT; 926 } 927 } else { 928 ts1.tv_sec = 30; 929 ts1.tv_nsec = 0; 930 for (;;) 931 __sys_nanosleep(&ts1, NULL); 932 } 933 break; 934 935 case PTHREAD_MUTEX_RECURSIVE: 936 /* Increment the lock count: */ 937 if (m->m_count + 1 > 0) { 938 m->m_count++; 939 ret = 0; 940 } else 941 ret = EAGAIN; 942 break; 943 944 default: 945 /* Trap invalid mutex types; */ 946 ret = EINVAL; 947 } 948 949 return (ret); 950 } 951 952 static __always_inline int 953 mutex_unlock_common(struct pthread_mutex *m, bool cv, int *mtx_defer) 954 { 955 struct pthread *curthread; 956 uint32_t id; 957 int deferred, error, private, robust; 958 959 if (__predict_false(m <= THR_MUTEX_DESTROYED)) { 960 if (m == THR_MUTEX_DESTROYED) 961 return (EINVAL); 962 return (EPERM); 963 } 964 965 curthread = _get_curthread(); 966 id = TID(curthread); 967 968 /* 969 * Check if the running thread is not the owner of the mutex. 970 */ 971 if (__predict_false(PMUTEX_OWNER_ID(m) != id)) 972 return (EPERM); 973 974 error = 0; 975 private = (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0; 976 if (__predict_false(PMUTEX_TYPE(m->m_flags) == 977 PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) { 978 m->m_count--; 979 } else { 980 if ((m->m_flags & PMUTEX_FLAG_DEFERRED) != 0) { 981 deferred = 1; 982 m->m_flags &= ~PMUTEX_FLAG_DEFERRED; 983 } else 984 deferred = 0; 985 986 robust = _mutex_enter_robust(curthread, m); 987 dequeue_mutex(curthread, m); 988 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer); 989 if (deferred) { 990 if (mtx_defer == NULL) { 991 _thr_wake_all(curthread->defer_waiters, 992 curthread->nwaiter_defer); 993 curthread->nwaiter_defer = 0; 994 } else 995 *mtx_defer = 1; 996 } 997 if (robust) 998 _mutex_leave_robust(curthread, m); 999 } 1000 if (!cv && private) 1001 THR_CRITICAL_LEAVE(curthread); 1002 return (error); 1003 } 1004 1005 int 1006 _pthread_mutex_getprioceiling(const pthread_mutex_t * __restrict mutex, 1007 int * __restrict prioceiling) 1008 { 1009 struct pthread_mutex *m; 1010 1011 if (*mutex == THR_PSHARED_PTR) { 1012 m = __thr_pshared_offpage(__DECONST(void *, mutex), 0); 1013 if (m == NULL) 1014 return (EINVAL); 1015 shared_mutex_init(m, NULL); 1016 } else { 1017 m = *mutex; 1018 if (m <= THR_MUTEX_DESTROYED) 1019 return (EINVAL); 1020 } 1021 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 1022 return (EINVAL); 1023 *prioceiling = m->m_lock.m_ceilings[0]; 1024 return (0); 1025 } 1026 1027 int 1028 _pthread_mutex_setprioceiling(pthread_mutex_t * __restrict mutex, 1029 int ceiling, int * __restrict old_ceiling) 1030 { 1031 struct pthread *curthread; 1032 struct pthread_mutex *m, *m1, *m2; 1033 struct mutex_queue *q, *qp; 1034 int qidx, ret; 1035 1036 if (*mutex == THR_PSHARED_PTR) { 1037 m = __thr_pshared_offpage(mutex, 0); 1038 if (m == NULL) 1039 return (EINVAL); 1040 shared_mutex_init(m, NULL); 1041 } else { 1042 m = *mutex; 1043 if (m <= THR_MUTEX_DESTROYED) 1044 return (EINVAL); 1045 } 1046 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 1047 return (EINVAL); 1048 1049 ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling); 1050 if (ret != 0) 1051 return (ret); 1052 1053 curthread = _get_curthread(); 1054 if (PMUTEX_OWNER_ID(m) == TID(curthread)) { 1055 mutex_assert_is_owned(m); 1056 m1 = TAILQ_PREV(m, mutex_queue, m_qe); 1057 m2 = TAILQ_NEXT(m, m_qe); 1058 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) || 1059 (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) { 1060 qidx = mutex_qidx(m); 1061 q = &curthread->mq[qidx]; 1062 qp = &curthread->mq[qidx + 1]; 1063 TAILQ_REMOVE(q, m, m_qe); 1064 if (!is_pshared_mutex(m)) 1065 TAILQ_REMOVE(qp, m, m_pqe); 1066 TAILQ_FOREACH(m2, q, m_qe) { 1067 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) { 1068 TAILQ_INSERT_BEFORE(m2, m, m_qe); 1069 if (!is_pshared_mutex(m)) { 1070 while (m2 != NULL && 1071 is_pshared_mutex(m2)) { 1072 m2 = TAILQ_PREV(m2, 1073 mutex_queue, m_qe); 1074 } 1075 if (m2 == NULL) { 1076 TAILQ_INSERT_HEAD(qp, 1077 m, m_pqe); 1078 } else { 1079 TAILQ_INSERT_BEFORE(m2, 1080 m, m_pqe); 1081 } 1082 } 1083 return (0); 1084 } 1085 } 1086 TAILQ_INSERT_TAIL(q, m, m_qe); 1087 if (!is_pshared_mutex(m)) 1088 TAILQ_INSERT_TAIL(qp, m, m_pqe); 1089 } 1090 } 1091 return (0); 1092 } 1093 1094 int 1095 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count) 1096 { 1097 struct pthread_mutex *m; 1098 int ret; 1099 1100 ret = check_and_init_mutex(mutex, &m); 1101 if (ret == 0) 1102 *count = m->m_spinloops; 1103 return (ret); 1104 } 1105 1106 int 1107 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count) 1108 { 1109 struct pthread_mutex *m; 1110 int ret; 1111 1112 ret = check_and_init_mutex(mutex, &m); 1113 if (ret == 0) 1114 m->m_spinloops = count; 1115 return (ret); 1116 } 1117 1118 int 1119 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count) 1120 { 1121 struct pthread_mutex *m; 1122 int ret; 1123 1124 ret = check_and_init_mutex(mutex, &m); 1125 if (ret == 0) 1126 *count = m->m_yieldloops; 1127 return (ret); 1128 } 1129 1130 int 1131 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count) 1132 { 1133 struct pthread_mutex *m; 1134 int ret; 1135 1136 ret = check_and_init_mutex(mutex, &m); 1137 if (ret == 0) 1138 m->m_yieldloops = count; 1139 return (0); 1140 } 1141 1142 int 1143 _pthread_mutex_isowned_np(pthread_mutex_t *mutex) 1144 { 1145 struct pthread_mutex *m; 1146 1147 if (*mutex == THR_PSHARED_PTR) { 1148 m = __thr_pshared_offpage(mutex, 0); 1149 if (m == NULL) 1150 return (0); 1151 shared_mutex_init(m, NULL); 1152 } else { 1153 m = *mutex; 1154 if (m <= THR_MUTEX_DESTROYED) 1155 return (0); 1156 } 1157 return (PMUTEX_OWNER_ID(m) == TID(_get_curthread())); 1158 } 1159 1160 int 1161 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp) 1162 { 1163 1164 if (__predict_false(mp <= THR_MUTEX_DESTROYED)) { 1165 if (mp == THR_MUTEX_DESTROYED) 1166 return (EINVAL); 1167 return (EPERM); 1168 } 1169 if (PMUTEX_OWNER_ID(mp) != TID(curthread)) 1170 return (EPERM); 1171 return (0); 1172 } 1173 1174 int 1175 _Tthr_mutex_consistent(pthread_mutex_t *mutex) 1176 { 1177 struct pthread_mutex *m; 1178 struct pthread *curthread; 1179 1180 if (*mutex == THR_PSHARED_PTR) { 1181 m = __thr_pshared_offpage(mutex, 0); 1182 if (m == NULL) 1183 return (EINVAL); 1184 shared_mutex_init(m, NULL); 1185 } else { 1186 m = *mutex; 1187 if (m <= THR_MUTEX_DESTROYED) 1188 return (EINVAL); 1189 } 1190 curthread = _get_curthread(); 1191 if ((m->m_lock.m_flags & (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 1192 (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) 1193 return (EINVAL); 1194 if (PMUTEX_OWNER_ID(m) != TID(curthread)) 1195 return (EPERM); 1196 m->m_lock.m_flags &= ~UMUTEX_NONCONSISTENT; 1197 return (0); 1198 } 1199