1 /* 2 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. 3 * Copyright (c) 2006 David Xu <davidxu@freebsd.org>. 4 * Copyright (c) 2015 The FreeBSD Foundation 5 * 6 * All rights reserved. 7 * 8 * Portions of this software were developed by Konstantin Belousov 9 * under sponsorship from the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by John Birrell. 22 * 4. Neither the name of the author nor the names of any co-contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * $FreeBSD$ 39 */ 40 41 #include <stdbool.h> 42 #include "namespace.h" 43 #include <stdlib.h> 44 #include <errno.h> 45 #include <string.h> 46 #include <sys/param.h> 47 #include <sys/queue.h> 48 #include <pthread.h> 49 #include <pthread_np.h> 50 #include "un-namespace.h" 51 52 #include "thr_private.h" 53 54 _Static_assert(sizeof(struct pthread_mutex) <= PAGE_SIZE, 55 "pthread_mutex is too large for off-page"); 56 57 /* 58 * For adaptive mutexes, how many times to spin doing trylock2 59 * before entering the kernel to block 60 */ 61 #define MUTEX_ADAPTIVE_SPINS 2000 62 63 /* 64 * Prototypes 65 */ 66 int __pthread_mutex_init(pthread_mutex_t *mutex, 67 const pthread_mutexattr_t *mutex_attr); 68 int __pthread_mutex_trylock(pthread_mutex_t *mutex); 69 int __pthread_mutex_lock(pthread_mutex_t *mutex); 70 int __pthread_mutex_timedlock(pthread_mutex_t *mutex, 71 const struct timespec *abstime); 72 int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 73 void *(calloc_cb)(size_t, size_t)); 74 int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count); 75 int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 76 int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 77 int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 78 int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count); 79 int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 80 81 static int mutex_self_trylock(pthread_mutex_t); 82 static int mutex_self_lock(pthread_mutex_t, 83 const struct timespec *abstime); 84 static int mutex_unlock_common(struct pthread_mutex *, int, int *); 85 static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, 86 const struct timespec *); 87 88 __weak_reference(__pthread_mutex_init, pthread_mutex_init); 89 __strong_reference(__pthread_mutex_init, _pthread_mutex_init); 90 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock); 91 __strong_reference(__pthread_mutex_lock, _pthread_mutex_lock); 92 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); 93 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock); 94 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); 95 __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock); 96 97 /* Single underscore versions provided for libc internal usage: */ 98 /* No difference between libc and application usage of these: */ 99 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy); 100 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock); 101 102 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling); 103 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling); 104 105 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np); 106 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np); 107 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np); 108 109 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np); 110 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np); 111 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np); 112 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np); 113 114 static void 115 mutex_init_link(struct pthread_mutex *m) 116 { 117 118 #if defined(_PTHREADS_INVARIANTS) 119 m->m_qe.tqe_prev = NULL; 120 m->m_qe.tqe_next = NULL; 121 m->m_pqe.tqe_prev = NULL; 122 m->m_pqe.tqe_next = NULL; 123 #endif 124 } 125 126 static void 127 mutex_assert_is_owned(struct pthread_mutex *m) 128 { 129 130 #if defined(_PTHREADS_INVARIANTS) 131 if (__predict_false(m->m_qe.tqe_prev == NULL)) { 132 char msg[128]; 133 snprintf(msg, sizeof(msg), 134 "mutex %p own %#x %#x is not on list %p %p", 135 m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, 136 m->m_qe.tqe_next); 137 PANIC(msg); 138 } 139 #endif 140 } 141 142 static void 143 mutex_assert_not_owned(struct pthread_mutex *m) 144 { 145 146 #if defined(_PTHREADS_INVARIANTS) 147 if (__predict_false(m->m_qe.tqe_prev != NULL || 148 m->m_qe.tqe_next != NULL)) { 149 char msg[128]; 150 snprintf(msg, sizeof(msg), 151 "mutex %p own %#x %#x is on list %p %p", 152 m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, 153 m->m_qe.tqe_next); 154 PANIC(msg); 155 } 156 #endif 157 } 158 159 static int 160 is_pshared_mutex(struct pthread_mutex *m) 161 { 162 163 return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0); 164 } 165 166 static int 167 mutex_check_attr(const struct pthread_mutex_attr *attr) 168 { 169 170 if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK || 171 attr->m_type >= PTHREAD_MUTEX_TYPE_MAX) 172 return (EINVAL); 173 if (attr->m_protocol < PTHREAD_PRIO_NONE || 174 attr->m_protocol > PTHREAD_PRIO_PROTECT) 175 return (EINVAL); 176 return (0); 177 } 178 179 static void 180 mutex_init_body(struct pthread_mutex *pmutex, 181 const struct pthread_mutex_attr *attr) 182 { 183 184 pmutex->m_flags = attr->m_type; 185 pmutex->m_owner = 0; 186 pmutex->m_count = 0; 187 pmutex->m_spinloops = 0; 188 pmutex->m_yieldloops = 0; 189 mutex_init_link(pmutex); 190 switch (attr->m_protocol) { 191 case PTHREAD_PRIO_NONE: 192 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 193 pmutex->m_lock.m_flags = 0; 194 break; 195 case PTHREAD_PRIO_INHERIT: 196 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 197 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT; 198 break; 199 case PTHREAD_PRIO_PROTECT: 200 pmutex->m_lock.m_owner = UMUTEX_CONTESTED; 201 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT; 202 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling; 203 break; 204 } 205 if (attr->m_pshared == PTHREAD_PROCESS_SHARED) 206 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED; 207 208 if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) { 209 pmutex->m_spinloops = 210 _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS; 211 pmutex->m_yieldloops = _thr_yieldloops; 212 } 213 } 214 215 static int 216 mutex_init(pthread_mutex_t *mutex, 217 const struct pthread_mutex_attr *mutex_attr, 218 void *(calloc_cb)(size_t, size_t)) 219 { 220 const struct pthread_mutex_attr *attr; 221 struct pthread_mutex *pmutex; 222 int error; 223 224 if (mutex_attr == NULL) { 225 attr = &_pthread_mutexattr_default; 226 } else { 227 attr = mutex_attr; 228 error = mutex_check_attr(attr); 229 if (error != 0) 230 return (error); 231 } 232 if ((pmutex = (pthread_mutex_t) 233 calloc_cb(1, sizeof(struct pthread_mutex))) == NULL) 234 return (ENOMEM); 235 mutex_init_body(pmutex, attr); 236 *mutex = pmutex; 237 return (0); 238 } 239 240 static int 241 init_static(struct pthread *thread, pthread_mutex_t *mutex) 242 { 243 int ret; 244 245 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 246 247 if (*mutex == THR_MUTEX_INITIALIZER) 248 ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc); 249 else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) 250 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, 251 calloc); 252 else 253 ret = 0; 254 THR_LOCK_RELEASE(thread, &_mutex_static_lock); 255 256 return (ret); 257 } 258 259 static void 260 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) 261 { 262 struct pthread_mutex *m2; 263 264 m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue); 265 if (m2 != NULL) 266 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0]; 267 else 268 m->m_lock.m_ceilings[1] = -1; 269 } 270 271 static void 272 shared_mutex_init(struct pthread_mutex *pmtx, const struct 273 pthread_mutex_attr *mutex_attr) 274 { 275 static const struct pthread_mutex_attr foobar_mutex_attr = { 276 .m_type = PTHREAD_MUTEX_DEFAULT, 277 .m_protocol = PTHREAD_PRIO_NONE, 278 .m_ceiling = 0, 279 .m_pshared = PTHREAD_PROCESS_SHARED 280 }; 281 bool done; 282 283 /* 284 * Hack to allow multiple pthread_mutex_init() calls on the 285 * same process-shared mutex. We rely on kernel allocating 286 * zeroed offpage for the mutex, i.e. the 287 * PMUTEX_INITSTAGE_ALLOC value must be zero. 288 */ 289 for (done = false; !done;) { 290 switch (pmtx->m_ps) { 291 case PMUTEX_INITSTAGE_DONE: 292 atomic_thread_fence_acq(); 293 done = true; 294 break; 295 case PMUTEX_INITSTAGE_ALLOC: 296 if (atomic_cmpset_int(&pmtx->m_ps, 297 PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) { 298 if (mutex_attr == NULL) 299 mutex_attr = &foobar_mutex_attr; 300 mutex_init_body(pmtx, mutex_attr); 301 atomic_store_rel_int(&pmtx->m_ps, 302 PMUTEX_INITSTAGE_DONE); 303 done = true; 304 } 305 break; 306 case PMUTEX_INITSTAGE_BUSY: 307 _pthread_yield(); 308 break; 309 default: 310 PANIC("corrupted offpage"); 311 break; 312 } 313 } 314 } 315 316 int 317 __pthread_mutex_init(pthread_mutex_t *mutex, 318 const pthread_mutexattr_t *mutex_attr) 319 { 320 struct pthread_mutex *pmtx; 321 int ret; 322 323 if (mutex_attr != NULL) { 324 ret = mutex_check_attr(*mutex_attr); 325 if (ret != 0) 326 return (ret); 327 } 328 if (mutex_attr == NULL || 329 (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { 330 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, 331 calloc)); 332 } 333 pmtx = __thr_pshared_offpage(mutex, 1); 334 if (pmtx == NULL) 335 return (EFAULT); 336 *mutex = THR_PSHARED_PTR; 337 shared_mutex_init(pmtx, *mutex_attr); 338 return (0); 339 } 340 341 /* This function is used internally by malloc. */ 342 int 343 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 344 void *(calloc_cb)(size_t, size_t)) 345 { 346 static const struct pthread_mutex_attr attr = { 347 .m_type = PTHREAD_MUTEX_NORMAL, 348 .m_protocol = PTHREAD_PRIO_NONE, 349 .m_ceiling = 0, 350 .m_pshared = PTHREAD_PROCESS_PRIVATE, 351 }; 352 int ret; 353 354 ret = mutex_init(mutex, &attr, calloc_cb); 355 if (ret == 0) 356 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE; 357 return (ret); 358 } 359 360 /* 361 * Fix mutex ownership for child process. 362 * 363 * Process private mutex ownership is transmitted from the forking 364 * thread to the child process. 365 * 366 * Process shared mutex should not be inherited because owner is 367 * forking thread which is in parent process, they are removed from 368 * the owned mutex list. 369 */ 370 static void 371 queue_fork(struct pthread *curthread, struct mutex_queue *q, 372 struct mutex_queue *qp, uint bit) 373 { 374 struct pthread_mutex *m; 375 376 TAILQ_INIT(q); 377 TAILQ_FOREACH(m, qp, m_pqe) { 378 TAILQ_INSERT_TAIL(q, m, m_qe); 379 m->m_lock.m_owner = TID(curthread) | bit; 380 m->m_owner = TID(curthread); 381 } 382 } 383 384 void 385 _mutex_fork(struct pthread *curthread) 386 { 387 388 queue_fork(curthread, &curthread->mq[TMQ_NORM], 389 &curthread->mq[TMQ_NORM_PRIV], 0); 390 queue_fork(curthread, &curthread->mq[TMQ_NORM_PP], 391 &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED); 392 } 393 394 int 395 _pthread_mutex_destroy(pthread_mutex_t *mutex) 396 { 397 pthread_mutex_t m, m1; 398 int ret; 399 400 m = *mutex; 401 if (m < THR_MUTEX_DESTROYED) { 402 ret = 0; 403 } else if (m == THR_MUTEX_DESTROYED) { 404 ret = EINVAL; 405 } else { 406 if (m == THR_PSHARED_PTR) { 407 m1 = __thr_pshared_offpage(mutex, 0); 408 if (m1 != NULL) { 409 mutex_assert_not_owned(m1); 410 __thr_pshared_destroy(mutex); 411 } 412 *mutex = THR_MUTEX_DESTROYED; 413 return (0); 414 } 415 if (m->m_owner != 0) { 416 ret = EBUSY; 417 } else { 418 *mutex = THR_MUTEX_DESTROYED; 419 mutex_assert_not_owned(m); 420 free(m); 421 ret = 0; 422 } 423 } 424 425 return (ret); 426 } 427 428 static int 429 mutex_qidx(struct pthread_mutex *m) 430 { 431 432 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 433 return (TMQ_NORM); 434 return (TMQ_NORM_PP); 435 } 436 437 static void 438 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m) 439 { 440 int qidx; 441 442 m->m_owner = TID(curthread); 443 /* Add to the list of owned mutexes: */ 444 mutex_assert_not_owned(m); 445 qidx = mutex_qidx(m); 446 TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe); 447 if (!is_pshared_mutex(m)) 448 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe); 449 } 450 451 static void 452 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m) 453 { 454 int qidx; 455 456 m->m_owner = 0; 457 mutex_assert_is_owned(m); 458 qidx = mutex_qidx(m); 459 TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe); 460 if (!is_pshared_mutex(m)) 461 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe); 462 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0) 463 set_inherited_priority(curthread, m); 464 mutex_init_link(m); 465 } 466 467 static int 468 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m) 469 { 470 int ret; 471 472 *m = *mutex; 473 ret = 0; 474 if (*m == THR_PSHARED_PTR) { 475 *m = __thr_pshared_offpage(mutex, 0); 476 if (*m == NULL) 477 ret = EINVAL; 478 shared_mutex_init(*m, NULL); 479 } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) { 480 if (*m == THR_MUTEX_DESTROYED) { 481 ret = EINVAL; 482 } else { 483 ret = init_static(_get_curthread(), mutex); 484 if (ret == 0) 485 *m = *mutex; 486 } 487 } 488 return (ret); 489 } 490 491 int 492 __pthread_mutex_trylock(pthread_mutex_t *mutex) 493 { 494 struct pthread *curthread; 495 struct pthread_mutex *m; 496 uint32_t id; 497 int ret; 498 499 ret = check_and_init_mutex(mutex, &m); 500 if (ret != 0) 501 return (ret); 502 curthread = _get_curthread(); 503 id = TID(curthread); 504 if (m->m_flags & PMUTEX_FLAG_PRIVATE) 505 THR_CRITICAL_ENTER(curthread); 506 ret = _thr_umutex_trylock(&m->m_lock, id); 507 if (__predict_true(ret == 0)) { 508 enqueue_mutex(curthread, m); 509 } else if (m->m_owner == id) { 510 ret = mutex_self_trylock(m); 511 } /* else {} */ 512 if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE)) 513 THR_CRITICAL_LEAVE(curthread); 514 return (ret); 515 } 516 517 static int 518 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, 519 const struct timespec *abstime) 520 { 521 uint32_t id, owner; 522 int count; 523 int ret; 524 525 id = TID(curthread); 526 if (m->m_owner == id) 527 return (mutex_self_lock(m, abstime)); 528 529 /* 530 * For adaptive mutexes, spin for a bit in the expectation 531 * that if the application requests this mutex type then 532 * the lock is likely to be released quickly and it is 533 * faster than entering the kernel 534 */ 535 if (__predict_false( 536 (m->m_lock.m_flags & 537 (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0)) 538 goto sleep_in_kernel; 539 540 if (!_thr_is_smp) 541 goto yield_loop; 542 543 count = m->m_spinloops; 544 while (count--) { 545 owner = m->m_lock.m_owner; 546 if ((owner & ~UMUTEX_CONTESTED) == 0) { 547 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { 548 ret = 0; 549 goto done; 550 } 551 } 552 CPU_SPINWAIT; 553 } 554 555 yield_loop: 556 count = m->m_yieldloops; 557 while (count--) { 558 _sched_yield(); 559 owner = m->m_lock.m_owner; 560 if ((owner & ~UMUTEX_CONTESTED) == 0) { 561 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { 562 ret = 0; 563 goto done; 564 } 565 } 566 } 567 568 sleep_in_kernel: 569 if (abstime == NULL) { 570 ret = __thr_umutex_lock(&m->m_lock, id); 571 } else if (__predict_false( 572 abstime->tv_nsec < 0 || 573 abstime->tv_nsec >= 1000000000)) { 574 ret = EINVAL; 575 } else { 576 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime); 577 } 578 done: 579 if (ret == 0) 580 enqueue_mutex(curthread, m); 581 582 return (ret); 583 } 584 585 static inline int 586 mutex_lock_common(struct pthread_mutex *m, 587 const struct timespec *abstime, int cvattach) 588 { 589 struct pthread *curthread = _get_curthread(); 590 int ret; 591 592 if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE) 593 THR_CRITICAL_ENTER(curthread); 594 if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) { 595 enqueue_mutex(curthread, m); 596 ret = 0; 597 } else { 598 ret = mutex_lock_sleep(curthread, m, abstime); 599 } 600 if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE) && !cvattach) 601 THR_CRITICAL_LEAVE(curthread); 602 return (ret); 603 } 604 605 int 606 __pthread_mutex_lock(pthread_mutex_t *mutex) 607 { 608 struct pthread_mutex *m; 609 int ret; 610 611 _thr_check_init(); 612 ret = check_and_init_mutex(mutex, &m); 613 if (ret == 0) 614 ret = mutex_lock_common(m, NULL, 0); 615 return (ret); 616 } 617 618 int 619 __pthread_mutex_timedlock(pthread_mutex_t *mutex, 620 const struct timespec *abstime) 621 { 622 struct pthread_mutex *m; 623 int ret; 624 625 _thr_check_init(); 626 ret = check_and_init_mutex(mutex, &m); 627 if (ret == 0) 628 ret = mutex_lock_common(m, abstime, 0); 629 return (ret); 630 } 631 632 int 633 _pthread_mutex_unlock(pthread_mutex_t *mutex) 634 { 635 struct pthread_mutex *mp; 636 637 if (*mutex == THR_PSHARED_PTR) { 638 mp = __thr_pshared_offpage(mutex, 0); 639 if (mp == NULL) 640 return (EINVAL); 641 shared_mutex_init(mp, NULL); 642 } else { 643 mp = *mutex; 644 } 645 return (mutex_unlock_common(mp, 0, NULL)); 646 } 647 648 int 649 _mutex_cv_lock(struct pthread_mutex *m, int count) 650 { 651 int error; 652 653 error = mutex_lock_common(m, NULL, 1); 654 if (error == 0) 655 m->m_count = count; 656 return (error); 657 } 658 659 int 660 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer) 661 { 662 663 /* 664 * Clear the count in case this is a recursive mutex. 665 */ 666 *count = m->m_count; 667 m->m_count = 0; 668 (void)mutex_unlock_common(m, 1, defer); 669 return (0); 670 } 671 672 int 673 _mutex_cv_attach(struct pthread_mutex *m, int count) 674 { 675 struct pthread *curthread = _get_curthread(); 676 677 enqueue_mutex(curthread, m); 678 m->m_count = count; 679 return (0); 680 } 681 682 int 683 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) 684 { 685 struct pthread *curthread = _get_curthread(); 686 int defered; 687 int error; 688 689 if ((error = _mutex_owned(curthread, mp)) != 0) 690 return (error); 691 692 /* 693 * Clear the count in case this is a recursive mutex. 694 */ 695 *recurse = mp->m_count; 696 mp->m_count = 0; 697 dequeue_mutex(curthread, mp); 698 699 /* Will this happen in real-world ? */ 700 if ((mp->m_flags & PMUTEX_FLAG_DEFERED) != 0) { 701 defered = 1; 702 mp->m_flags &= ~PMUTEX_FLAG_DEFERED; 703 } else 704 defered = 0; 705 706 if (defered) { 707 _thr_wake_all(curthread->defer_waiters, 708 curthread->nwaiter_defer); 709 curthread->nwaiter_defer = 0; 710 } 711 return (0); 712 } 713 714 static int 715 mutex_self_trylock(struct pthread_mutex *m) 716 { 717 int ret; 718 719 switch (PMUTEX_TYPE(m->m_flags)) { 720 case PTHREAD_MUTEX_ERRORCHECK: 721 case PTHREAD_MUTEX_NORMAL: 722 case PTHREAD_MUTEX_ADAPTIVE_NP: 723 ret = EBUSY; 724 break; 725 726 case PTHREAD_MUTEX_RECURSIVE: 727 /* Increment the lock count: */ 728 if (m->m_count + 1 > 0) { 729 m->m_count++; 730 ret = 0; 731 } else 732 ret = EAGAIN; 733 break; 734 735 default: 736 /* Trap invalid mutex types; */ 737 ret = EINVAL; 738 } 739 740 return (ret); 741 } 742 743 static int 744 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime) 745 { 746 struct timespec ts1, ts2; 747 int ret; 748 749 switch (PMUTEX_TYPE(m->m_flags)) { 750 case PTHREAD_MUTEX_ERRORCHECK: 751 case PTHREAD_MUTEX_ADAPTIVE_NP: 752 if (abstime) { 753 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 754 abstime->tv_nsec >= 1000000000) { 755 ret = EINVAL; 756 } else { 757 clock_gettime(CLOCK_REALTIME, &ts1); 758 TIMESPEC_SUB(&ts2, abstime, &ts1); 759 __sys_nanosleep(&ts2, NULL); 760 ret = ETIMEDOUT; 761 } 762 } else { 763 /* 764 * POSIX specifies that mutexes should return 765 * EDEADLK if a recursive lock is detected. 766 */ 767 ret = EDEADLK; 768 } 769 break; 770 771 case PTHREAD_MUTEX_NORMAL: 772 /* 773 * What SS2 define as a 'normal' mutex. Intentionally 774 * deadlock on attempts to get a lock you already own. 775 */ 776 ret = 0; 777 if (abstime) { 778 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 779 abstime->tv_nsec >= 1000000000) { 780 ret = EINVAL; 781 } else { 782 clock_gettime(CLOCK_REALTIME, &ts1); 783 TIMESPEC_SUB(&ts2, abstime, &ts1); 784 __sys_nanosleep(&ts2, NULL); 785 ret = ETIMEDOUT; 786 } 787 } else { 788 ts1.tv_sec = 30; 789 ts1.tv_nsec = 0; 790 for (;;) 791 __sys_nanosleep(&ts1, NULL); 792 } 793 break; 794 795 case PTHREAD_MUTEX_RECURSIVE: 796 /* Increment the lock count: */ 797 if (m->m_count + 1 > 0) { 798 m->m_count++; 799 ret = 0; 800 } else 801 ret = EAGAIN; 802 break; 803 804 default: 805 /* Trap invalid mutex types; */ 806 ret = EINVAL; 807 } 808 809 return (ret); 810 } 811 812 static int 813 mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer) 814 { 815 struct pthread *curthread = _get_curthread(); 816 uint32_t id; 817 int defered, error; 818 819 if (__predict_false(m <= THR_MUTEX_DESTROYED)) { 820 if (m == THR_MUTEX_DESTROYED) 821 return (EINVAL); 822 return (EPERM); 823 } 824 825 id = TID(curthread); 826 827 /* 828 * Check if the running thread is not the owner of the mutex. 829 */ 830 if (__predict_false(m->m_owner != id)) 831 return (EPERM); 832 833 error = 0; 834 if (__predict_false( 835 PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE && 836 m->m_count > 0)) { 837 m->m_count--; 838 } else { 839 if ((m->m_flags & PMUTEX_FLAG_DEFERED) != 0) { 840 defered = 1; 841 m->m_flags &= ~PMUTEX_FLAG_DEFERED; 842 } else 843 defered = 0; 844 845 dequeue_mutex(curthread, m); 846 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer); 847 848 if (mtx_defer == NULL && defered) { 849 _thr_wake_all(curthread->defer_waiters, 850 curthread->nwaiter_defer); 851 curthread->nwaiter_defer = 0; 852 } 853 } 854 if (!cv && m->m_flags & PMUTEX_FLAG_PRIVATE) 855 THR_CRITICAL_LEAVE(curthread); 856 return (error); 857 } 858 859 int 860 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex, 861 int *prioceiling) 862 { 863 struct pthread_mutex *m; 864 865 if (*mutex == THR_PSHARED_PTR) { 866 m = __thr_pshared_offpage(mutex, 0); 867 if (m == NULL) 868 return (EINVAL); 869 shared_mutex_init(m, NULL); 870 } else { 871 m = *mutex; 872 if (m <= THR_MUTEX_DESTROYED) 873 return (EINVAL); 874 } 875 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 876 return (EINVAL); 877 *prioceiling = m->m_lock.m_ceilings[0]; 878 return (0); 879 } 880 881 int 882 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex, 883 int ceiling, int *old_ceiling) 884 { 885 struct pthread *curthread; 886 struct pthread_mutex *m, *m1, *m2; 887 struct mutex_queue *q, *qp; 888 int ret; 889 890 if (*mutex == THR_PSHARED_PTR) { 891 m = __thr_pshared_offpage(mutex, 0); 892 if (m == NULL) 893 return (EINVAL); 894 shared_mutex_init(m, NULL); 895 } else { 896 m = *mutex; 897 if (m <= THR_MUTEX_DESTROYED) 898 return (EINVAL); 899 } 900 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 901 return (EINVAL); 902 903 ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling); 904 if (ret != 0) 905 return (ret); 906 907 curthread = _get_curthread(); 908 if (m->m_owner == TID(curthread)) { 909 mutex_assert_is_owned(m); 910 m1 = TAILQ_PREV(m, mutex_queue, m_qe); 911 m2 = TAILQ_NEXT(m, m_qe); 912 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) || 913 (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) { 914 q = &curthread->mq[TMQ_NORM_PP]; 915 qp = &curthread->mq[TMQ_NORM_PP_PRIV]; 916 TAILQ_REMOVE(q, m, m_qe); 917 if (!is_pshared_mutex(m)) 918 TAILQ_REMOVE(qp, m, m_pqe); 919 TAILQ_FOREACH(m2, q, m_qe) { 920 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) { 921 TAILQ_INSERT_BEFORE(m2, m, m_qe); 922 if (!is_pshared_mutex(m)) { 923 while (m2 != NULL && 924 is_pshared_mutex(m2)) { 925 m2 = TAILQ_PREV(m2, 926 mutex_queue, m_qe); 927 } 928 if (m2 == NULL) { 929 TAILQ_INSERT_HEAD(qp, 930 m, m_pqe); 931 } else { 932 TAILQ_INSERT_BEFORE(m2, 933 m, m_pqe); 934 } 935 } 936 return (0); 937 } 938 } 939 TAILQ_INSERT_TAIL(q, m, m_qe); 940 if (!is_pshared_mutex(m)) 941 TAILQ_INSERT_TAIL(qp, m, m_pqe); 942 } 943 } 944 return (0); 945 } 946 947 int 948 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count) 949 { 950 struct pthread_mutex *m; 951 int ret; 952 953 ret = check_and_init_mutex(mutex, &m); 954 if (ret == 0) 955 *count = m->m_spinloops; 956 return (ret); 957 } 958 959 int 960 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count) 961 { 962 struct pthread_mutex *m; 963 int ret; 964 965 ret = check_and_init_mutex(mutex, &m); 966 if (ret == 0) 967 m->m_spinloops = count; 968 return (ret); 969 } 970 971 int 972 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count) 973 { 974 struct pthread_mutex *m; 975 int ret; 976 977 ret = check_and_init_mutex(mutex, &m); 978 if (ret == 0) 979 *count = m->m_yieldloops; 980 return (ret); 981 } 982 983 int 984 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count) 985 { 986 struct pthread_mutex *m; 987 int ret; 988 989 ret = check_and_init_mutex(mutex, &m); 990 if (ret == 0) 991 m->m_yieldloops = count; 992 return (0); 993 } 994 995 int 996 _pthread_mutex_isowned_np(pthread_mutex_t *mutex) 997 { 998 struct pthread_mutex *m; 999 1000 if (*mutex == THR_PSHARED_PTR) { 1001 m = __thr_pshared_offpage(mutex, 0); 1002 if (m == NULL) 1003 return (0); 1004 shared_mutex_init(m, NULL); 1005 } else { 1006 m = *mutex; 1007 if (m <= THR_MUTEX_DESTROYED) 1008 return (0); 1009 } 1010 return (m->m_owner == TID(_get_curthread())); 1011 } 1012 1013 int 1014 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp) 1015 { 1016 if (__predict_false(mp <= THR_MUTEX_DESTROYED)) { 1017 if (mp == THR_MUTEX_DESTROYED) 1018 return (EINVAL); 1019 return (EPERM); 1020 } 1021 if (mp->m_owner != TID(curthread)) 1022 return (EPERM); 1023 return (0); 1024 } 1025