1 /* 2 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. 3 * Copyright (c) 2006 David Xu <davidxu@freebsd.org>. 4 * Copyright (c) 2015 The FreeBSD Foundation 5 * 6 * All rights reserved. 7 * 8 * Portions of this software were developed by Konstantin Belousov 9 * under sponsorship from the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by John Birrell. 22 * 4. Neither the name of the author nor the names of any co-contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include <sys/cdefs.h> 40 __FBSDID("$FreeBSD$"); 41 42 #include <stdbool.h> 43 #include "namespace.h" 44 #include <stdlib.h> 45 #include <errno.h> 46 #include <string.h> 47 #include <sys/param.h> 48 #include <sys/queue.h> 49 #include <pthread.h> 50 #include <pthread_np.h> 51 #include "un-namespace.h" 52 53 #include "thr_private.h" 54 55 _Static_assert(sizeof(struct pthread_mutex) <= PAGE_SIZE, 56 "pthread_mutex is too large for off-page"); 57 58 /* 59 * For adaptive mutexes, how many times to spin doing trylock2 60 * before entering the kernel to block 61 */ 62 #define MUTEX_ADAPTIVE_SPINS 2000 63 64 /* 65 * Prototypes 66 */ 67 int __pthread_mutex_init(pthread_mutex_t *mutex, 68 const pthread_mutexattr_t *mutex_attr); 69 int __pthread_mutex_trylock(pthread_mutex_t *mutex); 70 int __pthread_mutex_lock(pthread_mutex_t *mutex); 71 int __pthread_mutex_timedlock(pthread_mutex_t *mutex, 72 const struct timespec *abstime); 73 int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 74 void *(calloc_cb)(size_t, size_t)); 75 int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count); 76 int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 77 int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 78 int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 79 int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count); 80 int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 81 82 static int mutex_self_trylock(pthread_mutex_t); 83 static int mutex_self_lock(pthread_mutex_t, 84 const struct timespec *abstime); 85 static int mutex_unlock_common(struct pthread_mutex *, int, int *); 86 static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, 87 const struct timespec *); 88 89 __weak_reference(__pthread_mutex_init, pthread_mutex_init); 90 __strong_reference(__pthread_mutex_init, _pthread_mutex_init); 91 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock); 92 __strong_reference(__pthread_mutex_lock, _pthread_mutex_lock); 93 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); 94 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock); 95 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); 96 __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock); 97 98 /* Single underscore versions provided for libc internal usage: */ 99 /* No difference between libc and application usage of these: */ 100 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy); 101 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock); 102 103 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling); 104 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling); 105 106 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np); 107 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np); 108 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np); 109 110 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np); 111 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np); 112 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np); 113 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np); 114 115 static void 116 mutex_init_link(struct pthread_mutex *m) 117 { 118 119 #if defined(_PTHREADS_INVARIANTS) 120 m->m_qe.tqe_prev = NULL; 121 m->m_qe.tqe_next = NULL; 122 m->m_pqe.tqe_prev = NULL; 123 m->m_pqe.tqe_next = NULL; 124 #endif 125 } 126 127 static void 128 mutex_assert_is_owned(struct pthread_mutex *m) 129 { 130 131 #if defined(_PTHREADS_INVARIANTS) 132 if (__predict_false(m->m_qe.tqe_prev == NULL)) { 133 char msg[128]; 134 snprintf(msg, sizeof(msg), 135 "mutex %p own %#x %#x is not on list %p %p", 136 m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, 137 m->m_qe.tqe_next); 138 PANIC(msg); 139 } 140 #endif 141 } 142 143 static void 144 mutex_assert_not_owned(struct pthread_mutex *m) 145 { 146 147 #if defined(_PTHREADS_INVARIANTS) 148 if (__predict_false(m->m_qe.tqe_prev != NULL || 149 m->m_qe.tqe_next != NULL)) { 150 char msg[128]; 151 snprintf(msg, sizeof(msg), 152 "mutex %p own %#x %#x is on list %p %p", 153 m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, 154 m->m_qe.tqe_next); 155 PANIC(msg); 156 } 157 #endif 158 } 159 160 static int 161 is_pshared_mutex(struct pthread_mutex *m) 162 { 163 164 return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0); 165 } 166 167 static int 168 mutex_check_attr(const struct pthread_mutex_attr *attr) 169 { 170 171 if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK || 172 attr->m_type >= PTHREAD_MUTEX_TYPE_MAX) 173 return (EINVAL); 174 if (attr->m_protocol < PTHREAD_PRIO_NONE || 175 attr->m_protocol > PTHREAD_PRIO_PROTECT) 176 return (EINVAL); 177 return (0); 178 } 179 180 static void 181 mutex_init_body(struct pthread_mutex *pmutex, 182 const struct pthread_mutex_attr *attr) 183 { 184 185 pmutex->m_flags = attr->m_type; 186 pmutex->m_owner = 0; 187 pmutex->m_count = 0; 188 pmutex->m_spinloops = 0; 189 pmutex->m_yieldloops = 0; 190 mutex_init_link(pmutex); 191 switch (attr->m_protocol) { 192 case PTHREAD_PRIO_NONE: 193 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 194 pmutex->m_lock.m_flags = 0; 195 break; 196 case PTHREAD_PRIO_INHERIT: 197 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 198 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT; 199 break; 200 case PTHREAD_PRIO_PROTECT: 201 pmutex->m_lock.m_owner = UMUTEX_CONTESTED; 202 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT; 203 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling; 204 break; 205 } 206 if (attr->m_pshared == PTHREAD_PROCESS_SHARED) 207 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED; 208 209 if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) { 210 pmutex->m_spinloops = 211 _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS; 212 pmutex->m_yieldloops = _thr_yieldloops; 213 } 214 } 215 216 static int 217 mutex_init(pthread_mutex_t *mutex, 218 const struct pthread_mutex_attr *mutex_attr, 219 void *(calloc_cb)(size_t, size_t)) 220 { 221 const struct pthread_mutex_attr *attr; 222 struct pthread_mutex *pmutex; 223 int error; 224 225 if (mutex_attr == NULL) { 226 attr = &_pthread_mutexattr_default; 227 } else { 228 attr = mutex_attr; 229 error = mutex_check_attr(attr); 230 if (error != 0) 231 return (error); 232 } 233 if ((pmutex = (pthread_mutex_t) 234 calloc_cb(1, sizeof(struct pthread_mutex))) == NULL) 235 return (ENOMEM); 236 mutex_init_body(pmutex, attr); 237 *mutex = pmutex; 238 return (0); 239 } 240 241 static int 242 init_static(struct pthread *thread, pthread_mutex_t *mutex) 243 { 244 int ret; 245 246 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 247 248 if (*mutex == THR_MUTEX_INITIALIZER) 249 ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc); 250 else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) 251 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, 252 calloc); 253 else 254 ret = 0; 255 THR_LOCK_RELEASE(thread, &_mutex_static_lock); 256 257 return (ret); 258 } 259 260 static void 261 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) 262 { 263 struct pthread_mutex *m2; 264 265 m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue); 266 if (m2 != NULL) 267 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0]; 268 else 269 m->m_lock.m_ceilings[1] = -1; 270 } 271 272 static void 273 shared_mutex_init(struct pthread_mutex *pmtx, const struct 274 pthread_mutex_attr *mutex_attr) 275 { 276 static const struct pthread_mutex_attr foobar_mutex_attr = { 277 .m_type = PTHREAD_MUTEX_DEFAULT, 278 .m_protocol = PTHREAD_PRIO_NONE, 279 .m_ceiling = 0, 280 .m_pshared = PTHREAD_PROCESS_SHARED 281 }; 282 bool done; 283 284 /* 285 * Hack to allow multiple pthread_mutex_init() calls on the 286 * same process-shared mutex. We rely on kernel allocating 287 * zeroed offpage for the mutex, i.e. the 288 * PMUTEX_INITSTAGE_ALLOC value must be zero. 289 */ 290 for (done = false; !done;) { 291 switch (pmtx->m_ps) { 292 case PMUTEX_INITSTAGE_DONE: 293 atomic_thread_fence_acq(); 294 done = true; 295 break; 296 case PMUTEX_INITSTAGE_ALLOC: 297 if (atomic_cmpset_int(&pmtx->m_ps, 298 PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) { 299 if (mutex_attr == NULL) 300 mutex_attr = &foobar_mutex_attr; 301 mutex_init_body(pmtx, mutex_attr); 302 atomic_store_rel_int(&pmtx->m_ps, 303 PMUTEX_INITSTAGE_DONE); 304 done = true; 305 } 306 break; 307 case PMUTEX_INITSTAGE_BUSY: 308 _pthread_yield(); 309 break; 310 default: 311 PANIC("corrupted offpage"); 312 break; 313 } 314 } 315 } 316 317 int 318 __pthread_mutex_init(pthread_mutex_t *mutex, 319 const pthread_mutexattr_t *mutex_attr) 320 { 321 struct pthread_mutex *pmtx; 322 int ret; 323 324 if (mutex_attr != NULL) { 325 ret = mutex_check_attr(*mutex_attr); 326 if (ret != 0) 327 return (ret); 328 } 329 if (mutex_attr == NULL || 330 (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { 331 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, 332 calloc)); 333 } 334 pmtx = __thr_pshared_offpage(mutex, 1); 335 if (pmtx == NULL) 336 return (EFAULT); 337 *mutex = THR_PSHARED_PTR; 338 shared_mutex_init(pmtx, *mutex_attr); 339 return (0); 340 } 341 342 /* This function is used internally by malloc. */ 343 int 344 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 345 void *(calloc_cb)(size_t, size_t)) 346 { 347 static const struct pthread_mutex_attr attr = { 348 .m_type = PTHREAD_MUTEX_NORMAL, 349 .m_protocol = PTHREAD_PRIO_NONE, 350 .m_ceiling = 0, 351 .m_pshared = PTHREAD_PROCESS_PRIVATE, 352 }; 353 int ret; 354 355 ret = mutex_init(mutex, &attr, calloc_cb); 356 if (ret == 0) 357 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE; 358 return (ret); 359 } 360 361 /* 362 * Fix mutex ownership for child process. 363 * 364 * Process private mutex ownership is transmitted from the forking 365 * thread to the child process. 366 * 367 * Process shared mutex should not be inherited because owner is 368 * forking thread which is in parent process, they are removed from 369 * the owned mutex list. 370 */ 371 static void 372 queue_fork(struct pthread *curthread, struct mutex_queue *q, 373 struct mutex_queue *qp, uint bit) 374 { 375 struct pthread_mutex *m; 376 377 TAILQ_INIT(q); 378 TAILQ_FOREACH(m, qp, m_pqe) { 379 TAILQ_INSERT_TAIL(q, m, m_qe); 380 m->m_lock.m_owner = TID(curthread) | bit; 381 m->m_owner = TID(curthread); 382 } 383 } 384 385 void 386 _mutex_fork(struct pthread *curthread) 387 { 388 389 queue_fork(curthread, &curthread->mq[TMQ_NORM], 390 &curthread->mq[TMQ_NORM_PRIV], 0); 391 queue_fork(curthread, &curthread->mq[TMQ_NORM_PP], 392 &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED); 393 } 394 395 int 396 _pthread_mutex_destroy(pthread_mutex_t *mutex) 397 { 398 pthread_mutex_t m, m1; 399 int ret; 400 401 m = *mutex; 402 if (m < THR_MUTEX_DESTROYED) { 403 ret = 0; 404 } else if (m == THR_MUTEX_DESTROYED) { 405 ret = EINVAL; 406 } else { 407 if (m == THR_PSHARED_PTR) { 408 m1 = __thr_pshared_offpage(mutex, 0); 409 if (m1 != NULL) { 410 mutex_assert_not_owned(m1); 411 __thr_pshared_destroy(mutex); 412 } 413 *mutex = THR_MUTEX_DESTROYED; 414 return (0); 415 } 416 if (m->m_owner != 0) { 417 ret = EBUSY; 418 } else { 419 *mutex = THR_MUTEX_DESTROYED; 420 mutex_assert_not_owned(m); 421 free(m); 422 ret = 0; 423 } 424 } 425 426 return (ret); 427 } 428 429 static int 430 mutex_qidx(struct pthread_mutex *m) 431 { 432 433 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 434 return (TMQ_NORM); 435 return (TMQ_NORM_PP); 436 } 437 438 static void 439 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m) 440 { 441 int qidx; 442 443 m->m_owner = TID(curthread); 444 /* Add to the list of owned mutexes: */ 445 mutex_assert_not_owned(m); 446 qidx = mutex_qidx(m); 447 TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe); 448 if (!is_pshared_mutex(m)) 449 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe); 450 } 451 452 static void 453 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m) 454 { 455 int qidx; 456 457 m->m_owner = 0; 458 mutex_assert_is_owned(m); 459 qidx = mutex_qidx(m); 460 TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe); 461 if (!is_pshared_mutex(m)) 462 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe); 463 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0) 464 set_inherited_priority(curthread, m); 465 mutex_init_link(m); 466 } 467 468 static int 469 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m) 470 { 471 int ret; 472 473 *m = *mutex; 474 ret = 0; 475 if (*m == THR_PSHARED_PTR) { 476 *m = __thr_pshared_offpage(mutex, 0); 477 if (*m == NULL) 478 ret = EINVAL; 479 shared_mutex_init(*m, NULL); 480 } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) { 481 if (*m == THR_MUTEX_DESTROYED) { 482 ret = EINVAL; 483 } else { 484 ret = init_static(_get_curthread(), mutex); 485 if (ret == 0) 486 *m = *mutex; 487 } 488 } 489 return (ret); 490 } 491 492 int 493 __pthread_mutex_trylock(pthread_mutex_t *mutex) 494 { 495 struct pthread *curthread; 496 struct pthread_mutex *m; 497 uint32_t id; 498 int ret; 499 500 ret = check_and_init_mutex(mutex, &m); 501 if (ret != 0) 502 return (ret); 503 curthread = _get_curthread(); 504 id = TID(curthread); 505 if (m->m_flags & PMUTEX_FLAG_PRIVATE) 506 THR_CRITICAL_ENTER(curthread); 507 ret = _thr_umutex_trylock(&m->m_lock, id); 508 if (__predict_true(ret == 0)) { 509 enqueue_mutex(curthread, m); 510 } else if (m->m_owner == id) { 511 ret = mutex_self_trylock(m); 512 } /* else {} */ 513 if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE)) 514 THR_CRITICAL_LEAVE(curthread); 515 return (ret); 516 } 517 518 static int 519 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, 520 const struct timespec *abstime) 521 { 522 uint32_t id, owner; 523 int count; 524 int ret; 525 526 id = TID(curthread); 527 if (m->m_owner == id) 528 return (mutex_self_lock(m, abstime)); 529 530 /* 531 * For adaptive mutexes, spin for a bit in the expectation 532 * that if the application requests this mutex type then 533 * the lock is likely to be released quickly and it is 534 * faster than entering the kernel 535 */ 536 if (__predict_false( 537 (m->m_lock.m_flags & 538 (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0)) 539 goto sleep_in_kernel; 540 541 if (!_thr_is_smp) 542 goto yield_loop; 543 544 count = m->m_spinloops; 545 while (count--) { 546 owner = m->m_lock.m_owner; 547 if ((owner & ~UMUTEX_CONTESTED) == 0) { 548 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { 549 ret = 0; 550 goto done; 551 } 552 } 553 CPU_SPINWAIT; 554 } 555 556 yield_loop: 557 count = m->m_yieldloops; 558 while (count--) { 559 _sched_yield(); 560 owner = m->m_lock.m_owner; 561 if ((owner & ~UMUTEX_CONTESTED) == 0) { 562 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { 563 ret = 0; 564 goto done; 565 } 566 } 567 } 568 569 sleep_in_kernel: 570 if (abstime == NULL) { 571 ret = __thr_umutex_lock(&m->m_lock, id); 572 } else if (__predict_false( 573 abstime->tv_nsec < 0 || 574 abstime->tv_nsec >= 1000000000)) { 575 ret = EINVAL; 576 } else { 577 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime); 578 } 579 done: 580 if (ret == 0) 581 enqueue_mutex(curthread, m); 582 583 return (ret); 584 } 585 586 static inline int 587 mutex_lock_common(struct pthread_mutex *m, 588 const struct timespec *abstime, int cvattach) 589 { 590 struct pthread *curthread = _get_curthread(); 591 int ret; 592 593 if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE) 594 THR_CRITICAL_ENTER(curthread); 595 if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) { 596 enqueue_mutex(curthread, m); 597 ret = 0; 598 } else { 599 ret = mutex_lock_sleep(curthread, m, abstime); 600 } 601 if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE) && !cvattach) 602 THR_CRITICAL_LEAVE(curthread); 603 return (ret); 604 } 605 606 int 607 __pthread_mutex_lock(pthread_mutex_t *mutex) 608 { 609 struct pthread_mutex *m; 610 int ret; 611 612 _thr_check_init(); 613 ret = check_and_init_mutex(mutex, &m); 614 if (ret == 0) 615 ret = mutex_lock_common(m, NULL, 0); 616 return (ret); 617 } 618 619 int 620 __pthread_mutex_timedlock(pthread_mutex_t *mutex, 621 const struct timespec *abstime) 622 { 623 struct pthread_mutex *m; 624 int ret; 625 626 _thr_check_init(); 627 ret = check_and_init_mutex(mutex, &m); 628 if (ret == 0) 629 ret = mutex_lock_common(m, abstime, 0); 630 return (ret); 631 } 632 633 int 634 _pthread_mutex_unlock(pthread_mutex_t *mutex) 635 { 636 struct pthread_mutex *mp; 637 638 if (*mutex == THR_PSHARED_PTR) { 639 mp = __thr_pshared_offpage(mutex, 0); 640 if (mp == NULL) 641 return (EINVAL); 642 shared_mutex_init(mp, NULL); 643 } else { 644 mp = *mutex; 645 } 646 return (mutex_unlock_common(mp, 0, NULL)); 647 } 648 649 int 650 _mutex_cv_lock(struct pthread_mutex *m, int count) 651 { 652 int error; 653 654 error = mutex_lock_common(m, NULL, 1); 655 if (error == 0) 656 m->m_count = count; 657 return (error); 658 } 659 660 int 661 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer) 662 { 663 664 /* 665 * Clear the count in case this is a recursive mutex. 666 */ 667 *count = m->m_count; 668 m->m_count = 0; 669 (void)mutex_unlock_common(m, 1, defer); 670 return (0); 671 } 672 673 int 674 _mutex_cv_attach(struct pthread_mutex *m, int count) 675 { 676 struct pthread *curthread = _get_curthread(); 677 678 enqueue_mutex(curthread, m); 679 m->m_count = count; 680 return (0); 681 } 682 683 int 684 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) 685 { 686 struct pthread *curthread = _get_curthread(); 687 int defered; 688 int error; 689 690 if ((error = _mutex_owned(curthread, mp)) != 0) 691 return (error); 692 693 /* 694 * Clear the count in case this is a recursive mutex. 695 */ 696 *recurse = mp->m_count; 697 mp->m_count = 0; 698 dequeue_mutex(curthread, mp); 699 700 /* Will this happen in real-world ? */ 701 if ((mp->m_flags & PMUTEX_FLAG_DEFERED) != 0) { 702 defered = 1; 703 mp->m_flags &= ~PMUTEX_FLAG_DEFERED; 704 } else 705 defered = 0; 706 707 if (defered) { 708 _thr_wake_all(curthread->defer_waiters, 709 curthread->nwaiter_defer); 710 curthread->nwaiter_defer = 0; 711 } 712 return (0); 713 } 714 715 static int 716 mutex_self_trylock(struct pthread_mutex *m) 717 { 718 int ret; 719 720 switch (PMUTEX_TYPE(m->m_flags)) { 721 case PTHREAD_MUTEX_ERRORCHECK: 722 case PTHREAD_MUTEX_NORMAL: 723 case PTHREAD_MUTEX_ADAPTIVE_NP: 724 ret = EBUSY; 725 break; 726 727 case PTHREAD_MUTEX_RECURSIVE: 728 /* Increment the lock count: */ 729 if (m->m_count + 1 > 0) { 730 m->m_count++; 731 ret = 0; 732 } else 733 ret = EAGAIN; 734 break; 735 736 default: 737 /* Trap invalid mutex types; */ 738 ret = EINVAL; 739 } 740 741 return (ret); 742 } 743 744 static int 745 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime) 746 { 747 struct timespec ts1, ts2; 748 int ret; 749 750 switch (PMUTEX_TYPE(m->m_flags)) { 751 case PTHREAD_MUTEX_ERRORCHECK: 752 case PTHREAD_MUTEX_ADAPTIVE_NP: 753 if (abstime) { 754 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 755 abstime->tv_nsec >= 1000000000) { 756 ret = EINVAL; 757 } else { 758 clock_gettime(CLOCK_REALTIME, &ts1); 759 TIMESPEC_SUB(&ts2, abstime, &ts1); 760 __sys_nanosleep(&ts2, NULL); 761 ret = ETIMEDOUT; 762 } 763 } else { 764 /* 765 * POSIX specifies that mutexes should return 766 * EDEADLK if a recursive lock is detected. 767 */ 768 ret = EDEADLK; 769 } 770 break; 771 772 case PTHREAD_MUTEX_NORMAL: 773 /* 774 * What SS2 define as a 'normal' mutex. Intentionally 775 * deadlock on attempts to get a lock you already own. 776 */ 777 ret = 0; 778 if (abstime) { 779 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 780 abstime->tv_nsec >= 1000000000) { 781 ret = EINVAL; 782 } else { 783 clock_gettime(CLOCK_REALTIME, &ts1); 784 TIMESPEC_SUB(&ts2, abstime, &ts1); 785 __sys_nanosleep(&ts2, NULL); 786 ret = ETIMEDOUT; 787 } 788 } else { 789 ts1.tv_sec = 30; 790 ts1.tv_nsec = 0; 791 for (;;) 792 __sys_nanosleep(&ts1, NULL); 793 } 794 break; 795 796 case PTHREAD_MUTEX_RECURSIVE: 797 /* Increment the lock count: */ 798 if (m->m_count + 1 > 0) { 799 m->m_count++; 800 ret = 0; 801 } else 802 ret = EAGAIN; 803 break; 804 805 default: 806 /* Trap invalid mutex types; */ 807 ret = EINVAL; 808 } 809 810 return (ret); 811 } 812 813 static int 814 mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer) 815 { 816 struct pthread *curthread = _get_curthread(); 817 uint32_t id; 818 int defered, error; 819 820 if (__predict_false(m <= THR_MUTEX_DESTROYED)) { 821 if (m == THR_MUTEX_DESTROYED) 822 return (EINVAL); 823 return (EPERM); 824 } 825 826 id = TID(curthread); 827 828 /* 829 * Check if the running thread is not the owner of the mutex. 830 */ 831 if (__predict_false(m->m_owner != id)) 832 return (EPERM); 833 834 error = 0; 835 if (__predict_false( 836 PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE && 837 m->m_count > 0)) { 838 m->m_count--; 839 } else { 840 if ((m->m_flags & PMUTEX_FLAG_DEFERED) != 0) { 841 defered = 1; 842 m->m_flags &= ~PMUTEX_FLAG_DEFERED; 843 } else 844 defered = 0; 845 846 dequeue_mutex(curthread, m); 847 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer); 848 849 if (mtx_defer == NULL && defered) { 850 _thr_wake_all(curthread->defer_waiters, 851 curthread->nwaiter_defer); 852 curthread->nwaiter_defer = 0; 853 } 854 } 855 if (!cv && m->m_flags & PMUTEX_FLAG_PRIVATE) 856 THR_CRITICAL_LEAVE(curthread); 857 return (error); 858 } 859 860 int 861 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex, 862 int *prioceiling) 863 { 864 struct pthread_mutex *m; 865 866 if (*mutex == THR_PSHARED_PTR) { 867 m = __thr_pshared_offpage(mutex, 0); 868 if (m == NULL) 869 return (EINVAL); 870 shared_mutex_init(m, NULL); 871 } else { 872 m = *mutex; 873 if (m <= THR_MUTEX_DESTROYED) 874 return (EINVAL); 875 } 876 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 877 return (EINVAL); 878 *prioceiling = m->m_lock.m_ceilings[0]; 879 return (0); 880 } 881 882 int 883 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex, 884 int ceiling, int *old_ceiling) 885 { 886 struct pthread *curthread; 887 struct pthread_mutex *m, *m1, *m2; 888 struct mutex_queue *q, *qp; 889 int ret; 890 891 if (*mutex == THR_PSHARED_PTR) { 892 m = __thr_pshared_offpage(mutex, 0); 893 if (m == NULL) 894 return (EINVAL); 895 shared_mutex_init(m, NULL); 896 } else { 897 m = *mutex; 898 if (m <= THR_MUTEX_DESTROYED) 899 return (EINVAL); 900 } 901 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 902 return (EINVAL); 903 904 ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling); 905 if (ret != 0) 906 return (ret); 907 908 curthread = _get_curthread(); 909 if (m->m_owner == TID(curthread)) { 910 mutex_assert_is_owned(m); 911 m1 = TAILQ_PREV(m, mutex_queue, m_qe); 912 m2 = TAILQ_NEXT(m, m_qe); 913 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) || 914 (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) { 915 q = &curthread->mq[TMQ_NORM_PP]; 916 qp = &curthread->mq[TMQ_NORM_PP_PRIV]; 917 TAILQ_REMOVE(q, m, m_qe); 918 if (!is_pshared_mutex(m)) 919 TAILQ_REMOVE(qp, m, m_pqe); 920 TAILQ_FOREACH(m2, q, m_qe) { 921 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) { 922 TAILQ_INSERT_BEFORE(m2, m, m_qe); 923 if (!is_pshared_mutex(m)) { 924 while (m2 != NULL && 925 is_pshared_mutex(m2)) { 926 m2 = TAILQ_PREV(m2, 927 mutex_queue, m_qe); 928 } 929 if (m2 == NULL) { 930 TAILQ_INSERT_HEAD(qp, 931 m, m_pqe); 932 } else { 933 TAILQ_INSERT_BEFORE(m2, 934 m, m_pqe); 935 } 936 } 937 return (0); 938 } 939 } 940 TAILQ_INSERT_TAIL(q, m, m_qe); 941 if (!is_pshared_mutex(m)) 942 TAILQ_INSERT_TAIL(qp, m, m_pqe); 943 } 944 } 945 return (0); 946 } 947 948 int 949 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count) 950 { 951 struct pthread_mutex *m; 952 int ret; 953 954 ret = check_and_init_mutex(mutex, &m); 955 if (ret == 0) 956 *count = m->m_spinloops; 957 return (ret); 958 } 959 960 int 961 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count) 962 { 963 struct pthread_mutex *m; 964 int ret; 965 966 ret = check_and_init_mutex(mutex, &m); 967 if (ret == 0) 968 m->m_spinloops = count; 969 return (ret); 970 } 971 972 int 973 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count) 974 { 975 struct pthread_mutex *m; 976 int ret; 977 978 ret = check_and_init_mutex(mutex, &m); 979 if (ret == 0) 980 *count = m->m_yieldloops; 981 return (ret); 982 } 983 984 int 985 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count) 986 { 987 struct pthread_mutex *m; 988 int ret; 989 990 ret = check_and_init_mutex(mutex, &m); 991 if (ret == 0) 992 m->m_yieldloops = count; 993 return (0); 994 } 995 996 int 997 _pthread_mutex_isowned_np(pthread_mutex_t *mutex) 998 { 999 struct pthread_mutex *m; 1000 1001 if (*mutex == THR_PSHARED_PTR) { 1002 m = __thr_pshared_offpage(mutex, 0); 1003 if (m == NULL) 1004 return (0); 1005 shared_mutex_init(m, NULL); 1006 } else { 1007 m = *mutex; 1008 if (m <= THR_MUTEX_DESTROYED) 1009 return (0); 1010 } 1011 return (m->m_owner == TID(_get_curthread())); 1012 } 1013 1014 int 1015 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp) 1016 { 1017 if (__predict_false(mp <= THR_MUTEX_DESTROYED)) { 1018 if (mp == THR_MUTEX_DESTROYED) 1019 return (EINVAL); 1020 return (EPERM); 1021 } 1022 if (mp->m_owner != TID(curthread)) 1023 return (EPERM); 1024 return (0); 1025 } 1026