1 /* 2 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. 3 * Copyright (c) 2006 David Xu <davidxu@freebsd.org>. 4 * Copyright (c) 2015 The FreeBSD Foundation 5 * 6 * All rights reserved. 7 * 8 * Portions of this software were developed by Konstantin Belousov 9 * under sponsorship from the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by John Birrell. 22 * 4. Neither the name of the author nor the names of any co-contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * $FreeBSD$ 39 */ 40 41 #include "namespace.h" 42 #include <stdlib.h> 43 #include <errno.h> 44 #include <string.h> 45 #include <sys/param.h> 46 #include <sys/queue.h> 47 #include <pthread.h> 48 #include <pthread_np.h> 49 #include "un-namespace.h" 50 51 #include "thr_private.h" 52 53 /* 54 * For adaptive mutexes, how many times to spin doing trylock2 55 * before entering the kernel to block 56 */ 57 #define MUTEX_ADAPTIVE_SPINS 2000 58 59 /* 60 * Prototypes 61 */ 62 int __pthread_mutex_init(pthread_mutex_t *mutex, 63 const pthread_mutexattr_t *mutex_attr); 64 int __pthread_mutex_trylock(pthread_mutex_t *mutex); 65 int __pthread_mutex_lock(pthread_mutex_t *mutex); 66 int __pthread_mutex_timedlock(pthread_mutex_t *mutex, 67 const struct timespec *abstime); 68 int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 69 void *(calloc_cb)(size_t, size_t)); 70 int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count); 71 int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 72 int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count); 73 int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 74 int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count); 75 int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); 76 77 static int mutex_self_trylock(pthread_mutex_t); 78 static int mutex_self_lock(pthread_mutex_t, 79 const struct timespec *abstime); 80 static int mutex_unlock_common(struct pthread_mutex *, int, int *); 81 static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, 82 const struct timespec *); 83 84 __weak_reference(__pthread_mutex_init, pthread_mutex_init); 85 __strong_reference(__pthread_mutex_init, _pthread_mutex_init); 86 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock); 87 __strong_reference(__pthread_mutex_lock, _pthread_mutex_lock); 88 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); 89 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock); 90 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); 91 __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock); 92 93 /* Single underscore versions provided for libc internal usage: */ 94 /* No difference between libc and application usage of these: */ 95 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy); 96 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock); 97 98 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling); 99 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling); 100 101 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np); 102 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np); 103 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np); 104 105 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np); 106 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np); 107 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np); 108 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np); 109 110 static void 111 mutex_init_link(struct pthread_mutex *m) 112 { 113 114 #if defined(_PTHREADS_INVARIANTS) 115 m->m_qe.tqe_prev = NULL; 116 m->m_qe.tqe_next = NULL; 117 m->m_pqe.tqe_prev = NULL; 118 m->m_pqe.tqe_next = NULL; 119 #endif 120 } 121 122 static void 123 mutex_assert_is_owned(struct pthread_mutex *m) 124 { 125 126 #if defined(_PTHREADS_INVARIANTS) 127 if (__predict_false(m->m_qe.tqe_prev == NULL)) 128 PANIC("mutex is not on list"); 129 #endif 130 } 131 132 static void 133 mutex_assert_not_owned(struct pthread_mutex *m) 134 { 135 136 #if defined(_PTHREADS_INVARIANTS) 137 if (__predict_false(m->m_qe.tqe_prev != NULL || 138 m->m_qe.tqe_next != NULL)) 139 PANIC("mutex is on list"); 140 #endif 141 } 142 143 static int 144 is_pshared_mutex(struct pthread_mutex *m) 145 { 146 147 return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0); 148 } 149 150 static int 151 mutex_check_attr(const struct pthread_mutex_attr *attr) 152 { 153 154 if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK || 155 attr->m_type >= PTHREAD_MUTEX_TYPE_MAX) 156 return (EINVAL); 157 if (attr->m_protocol < PTHREAD_PRIO_NONE || 158 attr->m_protocol > PTHREAD_PRIO_PROTECT) 159 return (EINVAL); 160 return (0); 161 } 162 163 static void 164 mutex_init_body(struct pthread_mutex *pmutex, 165 const struct pthread_mutex_attr *attr) 166 { 167 168 pmutex->m_flags = attr->m_type; 169 pmutex->m_owner = 0; 170 pmutex->m_count = 0; 171 pmutex->m_spinloops = 0; 172 pmutex->m_yieldloops = 0; 173 mutex_init_link(pmutex); 174 switch (attr->m_protocol) { 175 case PTHREAD_PRIO_NONE: 176 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 177 pmutex->m_lock.m_flags = 0; 178 break; 179 case PTHREAD_PRIO_INHERIT: 180 pmutex->m_lock.m_owner = UMUTEX_UNOWNED; 181 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT; 182 break; 183 case PTHREAD_PRIO_PROTECT: 184 pmutex->m_lock.m_owner = UMUTEX_CONTESTED; 185 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT; 186 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling; 187 break; 188 } 189 if (attr->m_pshared == PTHREAD_PROCESS_SHARED) 190 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED; 191 192 if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) { 193 pmutex->m_spinloops = 194 _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS; 195 pmutex->m_yieldloops = _thr_yieldloops; 196 } 197 } 198 199 static int 200 mutex_init(pthread_mutex_t *mutex, 201 const struct pthread_mutex_attr *mutex_attr, 202 void *(calloc_cb)(size_t, size_t)) 203 { 204 const struct pthread_mutex_attr *attr; 205 struct pthread_mutex *pmutex; 206 int error; 207 208 if (mutex_attr == NULL) { 209 attr = &_pthread_mutexattr_default; 210 } else { 211 attr = mutex_attr; 212 error = mutex_check_attr(attr); 213 if (error != 0) 214 return (error); 215 } 216 if ((pmutex = (pthread_mutex_t) 217 calloc_cb(1, sizeof(struct pthread_mutex))) == NULL) 218 return (ENOMEM); 219 mutex_init_body(pmutex, attr); 220 *mutex = pmutex; 221 return (0); 222 } 223 224 static int 225 init_static(struct pthread *thread, pthread_mutex_t *mutex) 226 { 227 int ret; 228 229 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 230 231 if (*mutex == THR_MUTEX_INITIALIZER) 232 ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc); 233 else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) 234 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, 235 calloc); 236 else 237 ret = 0; 238 THR_LOCK_RELEASE(thread, &_mutex_static_lock); 239 240 return (ret); 241 } 242 243 static void 244 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) 245 { 246 struct pthread_mutex *m2; 247 248 m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue); 249 if (m2 != NULL) 250 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0]; 251 else 252 m->m_lock.m_ceilings[1] = -1; 253 } 254 255 int 256 __pthread_mutex_init(pthread_mutex_t *mutex, 257 const pthread_mutexattr_t *mutex_attr) 258 { 259 struct pthread_mutex *pmtx; 260 int ret; 261 262 if (mutex_attr != NULL) { 263 ret = mutex_check_attr(*mutex_attr); 264 if (ret != 0) 265 return (ret); 266 } 267 if (mutex_attr == NULL || 268 (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { 269 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, 270 calloc)); 271 } 272 pmtx = __thr_pshared_offpage(mutex, 1); 273 if (pmtx == NULL) 274 return (EFAULT); 275 *mutex = THR_PSHARED_PTR; 276 mutex_init_body(pmtx, *mutex_attr); 277 return (0); 278 } 279 280 /* This function is used internally by malloc. */ 281 int 282 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 283 void *(calloc_cb)(size_t, size_t)) 284 { 285 static const struct pthread_mutex_attr attr = { 286 .m_type = PTHREAD_MUTEX_NORMAL, 287 .m_protocol = PTHREAD_PRIO_NONE, 288 .m_ceiling = 0, 289 .m_pshared = PTHREAD_PROCESS_PRIVATE, 290 }; 291 int ret; 292 293 ret = mutex_init(mutex, &attr, calloc_cb); 294 if (ret == 0) 295 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE; 296 return (ret); 297 } 298 299 /* 300 * Fix mutex ownership for child process. 301 * 302 * Process private mutex ownership is transmitted from the forking 303 * thread to the child process. 304 * 305 * Process shared mutex should not be inherited because owner is 306 * forking thread which is in parent process, they are removed from 307 * the owned mutex list. 308 */ 309 static void 310 queue_fork(struct pthread *curthread, struct mutex_queue *q, 311 struct mutex_queue *qp, uint bit) 312 { 313 struct pthread_mutex *m; 314 315 TAILQ_INIT(q); 316 TAILQ_FOREACH(m, qp, m_pqe) { 317 TAILQ_INSERT_TAIL(q, m, m_qe); 318 m->m_lock.m_owner = TID(curthread) | bit; 319 m->m_owner = TID(curthread); 320 } 321 } 322 323 void 324 _mutex_fork(struct pthread *curthread) 325 { 326 327 queue_fork(curthread, &curthread->mq[TMQ_NORM], 328 &curthread->mq[TMQ_NORM_PRIV], 0); 329 queue_fork(curthread, &curthread->mq[TMQ_NORM_PP], 330 &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED); 331 } 332 333 int 334 _pthread_mutex_destroy(pthread_mutex_t *mutex) 335 { 336 pthread_mutex_t m, m1; 337 int ret; 338 339 m = *mutex; 340 if (m < THR_MUTEX_DESTROYED) { 341 ret = 0; 342 } else if (m == THR_MUTEX_DESTROYED) { 343 ret = EINVAL; 344 } else { 345 if (m == THR_PSHARED_PTR) { 346 m1 = __thr_pshared_offpage(mutex, 0); 347 if (m1 != NULL) { 348 mutex_assert_not_owned(m1); 349 __thr_pshared_destroy(mutex); 350 } 351 *mutex = THR_MUTEX_DESTROYED; 352 return (0); 353 } 354 if (m->m_owner != 0) { 355 ret = EBUSY; 356 } else { 357 *mutex = THR_MUTEX_DESTROYED; 358 mutex_assert_not_owned(m); 359 free(m); 360 ret = 0; 361 } 362 } 363 364 return (ret); 365 } 366 367 static int 368 mutex_qidx(struct pthread_mutex *m) 369 { 370 371 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 372 return (TMQ_NORM); 373 return (TMQ_NORM_PP); 374 } 375 376 static void 377 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m) 378 { 379 int qidx; 380 381 m->m_owner = TID(curthread); 382 /* Add to the list of owned mutexes: */ 383 mutex_assert_not_owned(m); 384 qidx = mutex_qidx(m); 385 TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe); 386 if (!is_pshared_mutex(m)) 387 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe); 388 } 389 390 static void 391 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m) 392 { 393 int qidx; 394 395 m->m_owner = 0; 396 mutex_assert_is_owned(m); 397 qidx = mutex_qidx(m); 398 TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe); 399 if (!is_pshared_mutex(m)) 400 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe); 401 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0) 402 set_inherited_priority(curthread, m); 403 mutex_init_link(m); 404 } 405 406 static int 407 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m) 408 { 409 int ret; 410 411 *m = *mutex; 412 ret = 0; 413 if (*m == THR_PSHARED_PTR) { 414 *m = __thr_pshared_offpage(mutex, 0); 415 if (*m == NULL) 416 ret = EINVAL; 417 } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) { 418 if (*m == THR_MUTEX_DESTROYED) { 419 ret = EINVAL; 420 } else { 421 ret = init_static(_get_curthread(), mutex); 422 if (ret == 0) 423 *m = *mutex; 424 } 425 } 426 return (ret); 427 } 428 429 int 430 __pthread_mutex_trylock(pthread_mutex_t *mutex) 431 { 432 struct pthread *curthread; 433 struct pthread_mutex *m; 434 uint32_t id; 435 int ret; 436 437 ret = check_and_init_mutex(mutex, &m); 438 if (ret != 0) 439 return (ret); 440 curthread = _get_curthread(); 441 id = TID(curthread); 442 if (m->m_flags & PMUTEX_FLAG_PRIVATE) 443 THR_CRITICAL_ENTER(curthread); 444 ret = _thr_umutex_trylock(&m->m_lock, id); 445 if (__predict_true(ret == 0)) { 446 enqueue_mutex(curthread, m); 447 } else if (m->m_owner == id) { 448 ret = mutex_self_trylock(m); 449 } /* else {} */ 450 if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE)) 451 THR_CRITICAL_LEAVE(curthread); 452 return (ret); 453 } 454 455 static int 456 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, 457 const struct timespec *abstime) 458 { 459 uint32_t id, owner; 460 int count; 461 int ret; 462 463 id = TID(curthread); 464 if (m->m_owner == id) 465 return (mutex_self_lock(m, abstime)); 466 467 /* 468 * For adaptive mutexes, spin for a bit in the expectation 469 * that if the application requests this mutex type then 470 * the lock is likely to be released quickly and it is 471 * faster than entering the kernel 472 */ 473 if (__predict_false( 474 (m->m_lock.m_flags & 475 (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0)) 476 goto sleep_in_kernel; 477 478 if (!_thr_is_smp) 479 goto yield_loop; 480 481 count = m->m_spinloops; 482 while (count--) { 483 owner = m->m_lock.m_owner; 484 if ((owner & ~UMUTEX_CONTESTED) == 0) { 485 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { 486 ret = 0; 487 goto done; 488 } 489 } 490 CPU_SPINWAIT; 491 } 492 493 yield_loop: 494 count = m->m_yieldloops; 495 while (count--) { 496 _sched_yield(); 497 owner = m->m_lock.m_owner; 498 if ((owner & ~UMUTEX_CONTESTED) == 0) { 499 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { 500 ret = 0; 501 goto done; 502 } 503 } 504 } 505 506 sleep_in_kernel: 507 if (abstime == NULL) { 508 ret = __thr_umutex_lock(&m->m_lock, id); 509 } else if (__predict_false( 510 abstime->tv_nsec < 0 || 511 abstime->tv_nsec >= 1000000000)) { 512 ret = EINVAL; 513 } else { 514 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime); 515 } 516 done: 517 if (ret == 0) 518 enqueue_mutex(curthread, m); 519 520 return (ret); 521 } 522 523 static inline int 524 mutex_lock_common(struct pthread_mutex *m, 525 const struct timespec *abstime, int cvattach) 526 { 527 struct pthread *curthread = _get_curthread(); 528 int ret; 529 530 if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE) 531 THR_CRITICAL_ENTER(curthread); 532 if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) { 533 enqueue_mutex(curthread, m); 534 ret = 0; 535 } else { 536 ret = mutex_lock_sleep(curthread, m, abstime); 537 } 538 if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE) && !cvattach) 539 THR_CRITICAL_LEAVE(curthread); 540 return (ret); 541 } 542 543 int 544 __pthread_mutex_lock(pthread_mutex_t *mutex) 545 { 546 struct pthread_mutex *m; 547 int ret; 548 549 _thr_check_init(); 550 ret = check_and_init_mutex(mutex, &m); 551 if (ret == 0) 552 ret = mutex_lock_common(m, NULL, 0); 553 return (ret); 554 } 555 556 int 557 __pthread_mutex_timedlock(pthread_mutex_t *mutex, 558 const struct timespec *abstime) 559 { 560 struct pthread_mutex *m; 561 int ret; 562 563 _thr_check_init(); 564 ret = check_and_init_mutex(mutex, &m); 565 if (ret == 0) 566 ret = mutex_lock_common(m, abstime, 0); 567 return (ret); 568 } 569 570 int 571 _pthread_mutex_unlock(pthread_mutex_t *mutex) 572 { 573 struct pthread_mutex *mp; 574 575 if (*mutex == THR_PSHARED_PTR) { 576 mp = __thr_pshared_offpage(mutex, 0); 577 if (mp == NULL) 578 return (EINVAL); 579 } else { 580 mp = *mutex; 581 } 582 return (mutex_unlock_common(mp, 0, NULL)); 583 } 584 585 int 586 _mutex_cv_lock(struct pthread_mutex *m, int count) 587 { 588 int error; 589 590 error = mutex_lock_common(m, NULL, 1); 591 if (error == 0) 592 m->m_count = count; 593 return (error); 594 } 595 596 int 597 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer) 598 { 599 600 /* 601 * Clear the count in case this is a recursive mutex. 602 */ 603 *count = m->m_count; 604 m->m_count = 0; 605 (void)mutex_unlock_common(m, 1, defer); 606 return (0); 607 } 608 609 int 610 _mutex_cv_attach(struct pthread_mutex *m, int count) 611 { 612 struct pthread *curthread = _get_curthread(); 613 614 enqueue_mutex(curthread, m); 615 m->m_count = count; 616 return (0); 617 } 618 619 int 620 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) 621 { 622 struct pthread *curthread = _get_curthread(); 623 int defered; 624 int error; 625 626 if ((error = _mutex_owned(curthread, mp)) != 0) 627 return (error); 628 629 /* 630 * Clear the count in case this is a recursive mutex. 631 */ 632 *recurse = mp->m_count; 633 mp->m_count = 0; 634 dequeue_mutex(curthread, mp); 635 636 /* Will this happen in real-world ? */ 637 if ((mp->m_flags & PMUTEX_FLAG_DEFERED) != 0) { 638 defered = 1; 639 mp->m_flags &= ~PMUTEX_FLAG_DEFERED; 640 } else 641 defered = 0; 642 643 if (defered) { 644 _thr_wake_all(curthread->defer_waiters, 645 curthread->nwaiter_defer); 646 curthread->nwaiter_defer = 0; 647 } 648 return (0); 649 } 650 651 static int 652 mutex_self_trylock(struct pthread_mutex *m) 653 { 654 int ret; 655 656 switch (PMUTEX_TYPE(m->m_flags)) { 657 case PTHREAD_MUTEX_ERRORCHECK: 658 case PTHREAD_MUTEX_NORMAL: 659 case PTHREAD_MUTEX_ADAPTIVE_NP: 660 ret = EBUSY; 661 break; 662 663 case PTHREAD_MUTEX_RECURSIVE: 664 /* Increment the lock count: */ 665 if (m->m_count + 1 > 0) { 666 m->m_count++; 667 ret = 0; 668 } else 669 ret = EAGAIN; 670 break; 671 672 default: 673 /* Trap invalid mutex types; */ 674 ret = EINVAL; 675 } 676 677 return (ret); 678 } 679 680 static int 681 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime) 682 { 683 struct timespec ts1, ts2; 684 int ret; 685 686 switch (PMUTEX_TYPE(m->m_flags)) { 687 case PTHREAD_MUTEX_ERRORCHECK: 688 case PTHREAD_MUTEX_ADAPTIVE_NP: 689 if (abstime) { 690 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 691 abstime->tv_nsec >= 1000000000) { 692 ret = EINVAL; 693 } else { 694 clock_gettime(CLOCK_REALTIME, &ts1); 695 TIMESPEC_SUB(&ts2, abstime, &ts1); 696 __sys_nanosleep(&ts2, NULL); 697 ret = ETIMEDOUT; 698 } 699 } else { 700 /* 701 * POSIX specifies that mutexes should return 702 * EDEADLK if a recursive lock is detected. 703 */ 704 ret = EDEADLK; 705 } 706 break; 707 708 case PTHREAD_MUTEX_NORMAL: 709 /* 710 * What SS2 define as a 'normal' mutex. Intentionally 711 * deadlock on attempts to get a lock you already own. 712 */ 713 ret = 0; 714 if (abstime) { 715 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 716 abstime->tv_nsec >= 1000000000) { 717 ret = EINVAL; 718 } else { 719 clock_gettime(CLOCK_REALTIME, &ts1); 720 TIMESPEC_SUB(&ts2, abstime, &ts1); 721 __sys_nanosleep(&ts2, NULL); 722 ret = ETIMEDOUT; 723 } 724 } else { 725 ts1.tv_sec = 30; 726 ts1.tv_nsec = 0; 727 for (;;) 728 __sys_nanosleep(&ts1, NULL); 729 } 730 break; 731 732 case PTHREAD_MUTEX_RECURSIVE: 733 /* Increment the lock count: */ 734 if (m->m_count + 1 > 0) { 735 m->m_count++; 736 ret = 0; 737 } else 738 ret = EAGAIN; 739 break; 740 741 default: 742 /* Trap invalid mutex types; */ 743 ret = EINVAL; 744 } 745 746 return (ret); 747 } 748 749 static int 750 mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer) 751 { 752 struct pthread *curthread = _get_curthread(); 753 uint32_t id; 754 int defered, error; 755 756 if (__predict_false(m <= THR_MUTEX_DESTROYED)) { 757 if (m == THR_MUTEX_DESTROYED) 758 return (EINVAL); 759 return (EPERM); 760 } 761 762 id = TID(curthread); 763 764 /* 765 * Check if the running thread is not the owner of the mutex. 766 */ 767 if (__predict_false(m->m_owner != id)) 768 return (EPERM); 769 770 error = 0; 771 if (__predict_false( 772 PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE && 773 m->m_count > 0)) { 774 m->m_count--; 775 } else { 776 if ((m->m_flags & PMUTEX_FLAG_DEFERED) != 0) { 777 defered = 1; 778 m->m_flags &= ~PMUTEX_FLAG_DEFERED; 779 } else 780 defered = 0; 781 782 dequeue_mutex(curthread, m); 783 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer); 784 785 if (mtx_defer == NULL && defered) { 786 _thr_wake_all(curthread->defer_waiters, 787 curthread->nwaiter_defer); 788 curthread->nwaiter_defer = 0; 789 } 790 } 791 if (!cv && m->m_flags & PMUTEX_FLAG_PRIVATE) 792 THR_CRITICAL_LEAVE(curthread); 793 return (error); 794 } 795 796 int 797 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex, 798 int *prioceiling) 799 { 800 struct pthread_mutex *m; 801 802 if (*mutex == THR_PSHARED_PTR) { 803 m = __thr_pshared_offpage(mutex, 0); 804 if (m == NULL) 805 return (EINVAL); 806 } else { 807 m = *mutex; 808 if (m <= THR_MUTEX_DESTROYED) 809 return (EINVAL); 810 } 811 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 812 return (EINVAL); 813 *prioceiling = m->m_lock.m_ceilings[0]; 814 return (0); 815 } 816 817 int 818 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex, 819 int ceiling, int *old_ceiling) 820 { 821 struct pthread *curthread; 822 struct pthread_mutex *m, *m1, *m2; 823 struct mutex_queue *q, *qp; 824 int ret; 825 826 if (*mutex == THR_PSHARED_PTR) { 827 m = __thr_pshared_offpage(mutex, 0); 828 if (m == NULL) 829 return (EINVAL); 830 } else { 831 m = *mutex; 832 if (m <= THR_MUTEX_DESTROYED) 833 return (EINVAL); 834 } 835 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) 836 return (EINVAL); 837 838 ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling); 839 if (ret != 0) 840 return (ret); 841 842 curthread = _get_curthread(); 843 if (m->m_owner == TID(curthread)) { 844 mutex_assert_is_owned(m); 845 m1 = TAILQ_PREV(m, mutex_queue, m_qe); 846 m2 = TAILQ_NEXT(m, m_qe); 847 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) || 848 (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) { 849 q = &curthread->mq[TMQ_NORM_PP]; 850 qp = &curthread->mq[TMQ_NORM_PP_PRIV]; 851 TAILQ_REMOVE(q, m, m_qe); 852 if (!is_pshared_mutex(m)) 853 TAILQ_REMOVE(qp, m, m_pqe); 854 TAILQ_FOREACH(m2, q, m_qe) { 855 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) { 856 TAILQ_INSERT_BEFORE(m2, m, m_qe); 857 if (!is_pshared_mutex(m)) { 858 while (m2 != NULL && 859 is_pshared_mutex(m2)) { 860 m2 = TAILQ_PREV(m2, 861 mutex_queue, m_qe); 862 } 863 if (m2 == NULL) { 864 TAILQ_INSERT_HEAD(qp, 865 m, m_pqe); 866 } else { 867 TAILQ_INSERT_BEFORE(m2, 868 m, m_pqe); 869 } 870 } 871 return (0); 872 } 873 } 874 TAILQ_INSERT_TAIL(q, m, m_qe); 875 if (!is_pshared_mutex(m)) 876 TAILQ_INSERT_TAIL(qp, m, m_pqe); 877 } 878 } 879 return (0); 880 } 881 882 int 883 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count) 884 { 885 struct pthread_mutex *m; 886 int ret; 887 888 ret = check_and_init_mutex(mutex, &m); 889 if (ret == 0) 890 *count = m->m_spinloops; 891 return (ret); 892 } 893 894 int 895 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count) 896 { 897 struct pthread_mutex *m; 898 int ret; 899 900 ret = check_and_init_mutex(mutex, &m); 901 if (ret == 0) 902 m->m_spinloops = count; 903 return (ret); 904 } 905 906 int 907 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count) 908 { 909 struct pthread_mutex *m; 910 int ret; 911 912 ret = check_and_init_mutex(mutex, &m); 913 if (ret == 0) 914 *count = m->m_yieldloops; 915 return (ret); 916 } 917 918 int 919 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count) 920 { 921 struct pthread_mutex *m; 922 int ret; 923 924 ret = check_and_init_mutex(mutex, &m); 925 if (ret == 0) 926 m->m_yieldloops = count; 927 return (0); 928 } 929 930 int 931 _pthread_mutex_isowned_np(pthread_mutex_t *mutex) 932 { 933 struct pthread_mutex *m; 934 935 if (*mutex == THR_PSHARED_PTR) { 936 m = __thr_pshared_offpage(mutex, 0); 937 if (m == NULL) 938 return (0); 939 } else { 940 m = *mutex; 941 if (m <= THR_MUTEX_DESTROYED) 942 return (0); 943 } 944 return (m->m_owner == TID(_get_curthread())); 945 } 946 947 int 948 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp) 949 { 950 if (__predict_false(mp <= THR_MUTEX_DESTROYED)) { 951 if (mp == THR_MUTEX_DESTROYED) 952 return (EINVAL); 953 return (EPERM); 954 } 955 if (mp->m_owner != TID(curthread)) 956 return (EPERM); 957 return (0); 958 } 959