1 /*- 2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice unmodified, this list of conditions, and the following 11 * disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_compat.h" 32 #include <sys/param.h> 33 #include <sys/kernel.h> 34 #include <sys/limits.h> 35 #include <sys/lock.h> 36 #include <sys/malloc.h> 37 #include <sys/mutex.h> 38 #include <sys/priv.h> 39 #include <sys/proc.h> 40 #include <sys/sched.h> 41 #include <sys/smp.h> 42 #include <sys/sysctl.h> 43 #include <sys/sysent.h> 44 #include <sys/systm.h> 45 #include <sys/sysproto.h> 46 #include <sys/eventhandler.h> 47 #include <sys/umtx.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_param.h> 51 #include <vm/pmap.h> 52 #include <vm/vm_map.h> 53 #include <vm/vm_object.h> 54 55 #include <machine/cpu.h> 56 57 #ifdef COMPAT_FREEBSD32 58 #include <compat/freebsd32/freebsd32_proto.h> 59 #endif 60 61 enum { 62 TYPE_SIMPLE_WAIT, 63 TYPE_CV, 64 TYPE_SEM, 65 TYPE_SIMPLE_LOCK, 66 TYPE_NORMAL_UMUTEX, 67 TYPE_PI_UMUTEX, 68 TYPE_PP_UMUTEX, 69 TYPE_RWLOCK 70 }; 71 72 #define _UMUTEX_TRY 1 73 #define _UMUTEX_WAIT 2 74 75 /* Key to represent a unique userland synchronous object */ 76 struct umtx_key { 77 int hash; 78 int type; 79 int shared; 80 union { 81 struct { 82 vm_object_t object; 83 uintptr_t offset; 84 } shared; 85 struct { 86 struct vmspace *vs; 87 uintptr_t addr; 88 } private; 89 struct { 90 void *a; 91 uintptr_t b; 92 } both; 93 } info; 94 }; 95 96 /* Priority inheritance mutex info. */ 97 struct umtx_pi { 98 /* Owner thread */ 99 struct thread *pi_owner; 100 101 /* Reference count */ 102 int pi_refcount; 103 104 /* List entry to link umtx holding by thread */ 105 TAILQ_ENTRY(umtx_pi) pi_link; 106 107 /* List entry in hash */ 108 TAILQ_ENTRY(umtx_pi) pi_hashlink; 109 110 /* List for waiters */ 111 TAILQ_HEAD(,umtx_q) pi_blocked; 112 113 /* Identify a userland lock object */ 114 struct umtx_key pi_key; 115 }; 116 117 /* A userland synchronous object user. */ 118 struct umtx_q { 119 /* Linked list for the hash. */ 120 TAILQ_ENTRY(umtx_q) uq_link; 121 122 /* Umtx key. */ 123 struct umtx_key uq_key; 124 125 /* Umtx flags. */ 126 int uq_flags; 127 #define UQF_UMTXQ 0x0001 128 129 /* The thread waits on. */ 130 struct thread *uq_thread; 131 132 /* 133 * Blocked on PI mutex. read can use chain lock 134 * or umtx_lock, write must have both chain lock and 135 * umtx_lock being hold. 136 */ 137 struct umtx_pi *uq_pi_blocked; 138 139 /* On blocked list */ 140 TAILQ_ENTRY(umtx_q) uq_lockq; 141 142 /* Thread contending with us */ 143 TAILQ_HEAD(,umtx_pi) uq_pi_contested; 144 145 /* Inherited priority from PP mutex */ 146 u_char uq_inherited_pri; 147 148 /* Spare queue ready to be reused */ 149 struct umtxq_queue *uq_spare_queue; 150 151 /* The queue we on */ 152 struct umtxq_queue *uq_cur_queue; 153 }; 154 155 TAILQ_HEAD(umtxq_head, umtx_q); 156 157 /* Per-key wait-queue */ 158 struct umtxq_queue { 159 struct umtxq_head head; 160 struct umtx_key key; 161 LIST_ENTRY(umtxq_queue) link; 162 int length; 163 }; 164 165 LIST_HEAD(umtxq_list, umtxq_queue); 166 167 /* Userland lock object's wait-queue chain */ 168 struct umtxq_chain { 169 /* Lock for this chain. */ 170 struct mtx uc_lock; 171 172 /* List of sleep queues. */ 173 struct umtxq_list uc_queue[2]; 174 #define UMTX_SHARED_QUEUE 0 175 #define UMTX_EXCLUSIVE_QUEUE 1 176 177 LIST_HEAD(, umtxq_queue) uc_spare_queue; 178 179 /* Busy flag */ 180 char uc_busy; 181 182 /* Chain lock waiters */ 183 int uc_waiters; 184 185 /* All PI in the list */ 186 TAILQ_HEAD(,umtx_pi) uc_pi_list; 187 188 }; 189 190 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) 191 #define UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy")) 192 193 /* 194 * Don't propagate time-sharing priority, there is a security reason, 195 * a user can simply introduce PI-mutex, let thread A lock the mutex, 196 * and let another thread B block on the mutex, because B is 197 * sleeping, its priority will be boosted, this causes A's priority to 198 * be boosted via priority propagating too and will never be lowered even 199 * if it is using 100%CPU, this is unfair to other processes. 200 */ 201 202 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ 203 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ 204 PRI_MAX_TIMESHARE : (td)->td_user_pri) 205 206 #define GOLDEN_RATIO_PRIME 2654404609U 207 #define UMTX_CHAINS 128 208 #define UMTX_SHIFTS (__WORD_BIT - 7) 209 210 #define THREAD_SHARE 0 211 #define PROCESS_SHARE 1 212 #define AUTO_SHARE 2 213 214 #define GET_SHARE(flags) \ 215 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) 216 217 #define BUSY_SPINS 200 218 219 static uma_zone_t umtx_pi_zone; 220 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; 221 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); 222 static int umtx_pi_allocated; 223 224 SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); 225 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, 226 &umtx_pi_allocated, 0, "Allocated umtx_pi"); 227 228 static void umtxq_sysinit(void *); 229 static void umtxq_hash(struct umtx_key *key); 230 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); 231 static void umtxq_lock(struct umtx_key *key); 232 static void umtxq_unlock(struct umtx_key *key); 233 static void umtxq_busy(struct umtx_key *key); 234 static void umtxq_unbusy(struct umtx_key *key); 235 static void umtxq_insert_queue(struct umtx_q *uq, int q); 236 static void umtxq_remove_queue(struct umtx_q *uq, int q); 237 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo); 238 static int umtxq_count(struct umtx_key *key); 239 static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2); 240 static int umtx_key_get(void *addr, int type, int share, 241 struct umtx_key *key); 242 static void umtx_key_release(struct umtx_key *key); 243 static struct umtx_pi *umtx_pi_alloc(int); 244 static void umtx_pi_free(struct umtx_pi *pi); 245 static void umtx_pi_adjust_locked(struct thread *td, u_char oldpri); 246 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags); 247 static void umtx_thread_cleanup(struct thread *td); 248 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, 249 struct image_params *imgp __unused); 250 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); 251 252 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE) 253 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE) 254 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE) 255 256 static struct mtx umtx_lock; 257 258 static void 259 umtxq_sysinit(void *arg __unused) 260 { 261 int i, j; 262 263 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), 264 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 265 for (i = 0; i < 2; ++i) { 266 for (j = 0; j < UMTX_CHAINS; ++j) { 267 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL, 268 MTX_DEF | MTX_DUPOK); 269 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]); 270 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]); 271 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue); 272 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list); 273 umtxq_chains[i][j].uc_busy = 0; 274 umtxq_chains[i][j].uc_waiters = 0; 275 } 276 } 277 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN); 278 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, 279 EVENTHANDLER_PRI_ANY); 280 } 281 282 struct umtx_q * 283 umtxq_alloc(void) 284 { 285 struct umtx_q *uq; 286 287 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); 288 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, M_WAITOK | M_ZERO); 289 TAILQ_INIT(&uq->uq_spare_queue->head); 290 TAILQ_INIT(&uq->uq_pi_contested); 291 uq->uq_inherited_pri = PRI_MAX; 292 return (uq); 293 } 294 295 void 296 umtxq_free(struct umtx_q *uq) 297 { 298 MPASS(uq->uq_spare_queue != NULL); 299 free(uq->uq_spare_queue, M_UMTX); 300 free(uq, M_UMTX); 301 } 302 303 static inline void 304 umtxq_hash(struct umtx_key *key) 305 { 306 unsigned n = (uintptr_t)key->info.both.a + key->info.both.b; 307 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; 308 } 309 310 static inline int 311 umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) 312 { 313 return (k1->type == k2->type && 314 k1->info.both.a == k2->info.both.a && 315 k1->info.both.b == k2->info.both.b); 316 } 317 318 static inline struct umtxq_chain * 319 umtxq_getchain(struct umtx_key *key) 320 { 321 if (key->type <= TYPE_SEM) 322 return (&umtxq_chains[1][key->hash]); 323 return (&umtxq_chains[0][key->hash]); 324 } 325 326 /* 327 * Lock a chain. 328 */ 329 static inline void 330 umtxq_lock(struct umtx_key *key) 331 { 332 struct umtxq_chain *uc; 333 334 uc = umtxq_getchain(key); 335 mtx_lock(&uc->uc_lock); 336 } 337 338 /* 339 * Unlock a chain. 340 */ 341 static inline void 342 umtxq_unlock(struct umtx_key *key) 343 { 344 struct umtxq_chain *uc; 345 346 uc = umtxq_getchain(key); 347 mtx_unlock(&uc->uc_lock); 348 } 349 350 /* 351 * Set chain to busy state when following operation 352 * may be blocked (kernel mutex can not be used). 353 */ 354 static inline void 355 umtxq_busy(struct umtx_key *key) 356 { 357 struct umtxq_chain *uc; 358 359 uc = umtxq_getchain(key); 360 mtx_assert(&uc->uc_lock, MA_OWNED); 361 if (uc->uc_busy) { 362 #ifdef SMP 363 if (smp_cpus > 1) { 364 int count = BUSY_SPINS; 365 if (count > 0) { 366 umtxq_unlock(key); 367 while (uc->uc_busy && --count > 0) 368 cpu_spinwait(); 369 umtxq_lock(key); 370 } 371 } 372 #endif 373 while (uc->uc_busy) { 374 uc->uc_waiters++; 375 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); 376 uc->uc_waiters--; 377 } 378 } 379 uc->uc_busy = 1; 380 } 381 382 /* 383 * Unbusy a chain. 384 */ 385 static inline void 386 umtxq_unbusy(struct umtx_key *key) 387 { 388 struct umtxq_chain *uc; 389 390 uc = umtxq_getchain(key); 391 mtx_assert(&uc->uc_lock, MA_OWNED); 392 KASSERT(uc->uc_busy != 0, ("not busy")); 393 uc->uc_busy = 0; 394 if (uc->uc_waiters) 395 wakeup_one(uc); 396 } 397 398 static struct umtxq_queue * 399 umtxq_queue_lookup(struct umtx_key *key, int q) 400 { 401 struct umtxq_queue *uh; 402 struct umtxq_chain *uc; 403 404 uc = umtxq_getchain(key); 405 UMTXQ_LOCKED_ASSERT(uc); 406 LIST_FOREACH(uh, &uc->uc_queue[q], link) { 407 if (umtx_key_match(&uh->key, key)) 408 return (uh); 409 } 410 411 return (NULL); 412 } 413 414 static inline void 415 umtxq_insert_queue(struct umtx_q *uq, int q) 416 { 417 struct umtxq_queue *uh; 418 struct umtxq_chain *uc; 419 420 uc = umtxq_getchain(&uq->uq_key); 421 UMTXQ_LOCKED_ASSERT(uc); 422 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue")); 423 uh = umtxq_queue_lookup(&uq->uq_key, q); 424 if (uh != NULL) { 425 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link); 426 } else { 427 uh = uq->uq_spare_queue; 428 uh->key = uq->uq_key; 429 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link); 430 } 431 uq->uq_spare_queue = NULL; 432 433 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link); 434 uh->length++; 435 uq->uq_flags |= UQF_UMTXQ; 436 uq->uq_cur_queue = uh; 437 return; 438 } 439 440 static inline void 441 umtxq_remove_queue(struct umtx_q *uq, int q) 442 { 443 struct umtxq_chain *uc; 444 struct umtxq_queue *uh; 445 446 uc = umtxq_getchain(&uq->uq_key); 447 UMTXQ_LOCKED_ASSERT(uc); 448 if (uq->uq_flags & UQF_UMTXQ) { 449 uh = uq->uq_cur_queue; 450 TAILQ_REMOVE(&uh->head, uq, uq_link); 451 uh->length--; 452 uq->uq_flags &= ~UQF_UMTXQ; 453 if (TAILQ_EMPTY(&uh->head)) { 454 KASSERT(uh->length == 0, 455 ("inconsistent umtxq_queue length")); 456 LIST_REMOVE(uh, link); 457 } else { 458 uh = LIST_FIRST(&uc->uc_spare_queue); 459 KASSERT(uh != NULL, ("uc_spare_queue is empty")); 460 LIST_REMOVE(uh, link); 461 } 462 uq->uq_spare_queue = uh; 463 uq->uq_cur_queue = NULL; 464 } 465 } 466 467 /* 468 * Check if there are multiple waiters 469 */ 470 static int 471 umtxq_count(struct umtx_key *key) 472 { 473 struct umtxq_chain *uc; 474 struct umtxq_queue *uh; 475 476 uc = umtxq_getchain(key); 477 UMTXQ_LOCKED_ASSERT(uc); 478 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); 479 if (uh != NULL) 480 return (uh->length); 481 return (0); 482 } 483 484 /* 485 * Check if there are multiple PI waiters and returns first 486 * waiter. 487 */ 488 static int 489 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) 490 { 491 struct umtxq_chain *uc; 492 struct umtxq_queue *uh; 493 494 *first = NULL; 495 uc = umtxq_getchain(key); 496 UMTXQ_LOCKED_ASSERT(uc); 497 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); 498 if (uh != NULL) { 499 *first = TAILQ_FIRST(&uh->head); 500 return (uh->length); 501 } 502 return (0); 503 } 504 505 /* 506 * Wake up threads waiting on an userland object. 507 */ 508 509 static int 510 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q) 511 { 512 struct umtxq_chain *uc; 513 struct umtxq_queue *uh; 514 struct umtx_q *uq; 515 int ret; 516 517 ret = 0; 518 uc = umtxq_getchain(key); 519 UMTXQ_LOCKED_ASSERT(uc); 520 uh = umtxq_queue_lookup(key, q); 521 if (uh != NULL) { 522 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) { 523 umtxq_remove_queue(uq, q); 524 wakeup(uq); 525 if (++ret >= n_wake) 526 return (ret); 527 } 528 } 529 return (ret); 530 } 531 532 533 /* 534 * Wake up specified thread. 535 */ 536 static inline void 537 umtxq_signal_thread(struct umtx_q *uq) 538 { 539 struct umtxq_chain *uc; 540 541 uc = umtxq_getchain(&uq->uq_key); 542 UMTXQ_LOCKED_ASSERT(uc); 543 umtxq_remove(uq); 544 wakeup(uq); 545 } 546 547 /* 548 * Put thread into sleep state, before sleeping, check if 549 * thread was removed from umtx queue. 550 */ 551 static inline int 552 umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo) 553 { 554 struct umtxq_chain *uc; 555 int error; 556 557 uc = umtxq_getchain(&uq->uq_key); 558 UMTXQ_LOCKED_ASSERT(uc); 559 if (!(uq->uq_flags & UQF_UMTXQ)) 560 return (0); 561 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); 562 if (error == EWOULDBLOCK) 563 error = ETIMEDOUT; 564 return (error); 565 } 566 567 /* 568 * Convert userspace address into unique logical address. 569 */ 570 static int 571 umtx_key_get(void *addr, int type, int share, struct umtx_key *key) 572 { 573 struct thread *td = curthread; 574 vm_map_t map; 575 vm_map_entry_t entry; 576 vm_pindex_t pindex; 577 vm_prot_t prot; 578 boolean_t wired; 579 580 key->type = type; 581 if (share == THREAD_SHARE) { 582 key->shared = 0; 583 key->info.private.vs = td->td_proc->p_vmspace; 584 key->info.private.addr = (uintptr_t)addr; 585 } else { 586 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE); 587 map = &td->td_proc->p_vmspace->vm_map; 588 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE, 589 &entry, &key->info.shared.object, &pindex, &prot, 590 &wired) != KERN_SUCCESS) { 591 return EFAULT; 592 } 593 594 if ((share == PROCESS_SHARE) || 595 (share == AUTO_SHARE && 596 VM_INHERIT_SHARE == entry->inheritance)) { 597 key->shared = 1; 598 key->info.shared.offset = entry->offset + entry->start - 599 (vm_offset_t)addr; 600 vm_object_reference(key->info.shared.object); 601 } else { 602 key->shared = 0; 603 key->info.private.vs = td->td_proc->p_vmspace; 604 key->info.private.addr = (uintptr_t)addr; 605 } 606 vm_map_lookup_done(map, entry); 607 } 608 609 umtxq_hash(key); 610 return (0); 611 } 612 613 /* 614 * Release key. 615 */ 616 static inline void 617 umtx_key_release(struct umtx_key *key) 618 { 619 if (key->shared) 620 vm_object_deallocate(key->info.shared.object); 621 } 622 623 /* 624 * Lock a umtx object. 625 */ 626 static int 627 _do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo) 628 { 629 struct umtx_q *uq; 630 u_long owner; 631 u_long old; 632 int error = 0; 633 634 uq = td->td_umtxq; 635 636 /* 637 * Care must be exercised when dealing with umtx structure. It 638 * can fault on any access. 639 */ 640 for (;;) { 641 /* 642 * Try the uncontested case. This should be done in userland. 643 */ 644 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id); 645 646 /* The acquire succeeded. */ 647 if (owner == UMTX_UNOWNED) 648 return (0); 649 650 /* The address was invalid. */ 651 if (owner == -1) 652 return (EFAULT); 653 654 /* If no one owns it but it is contested try to acquire it. */ 655 if (owner == UMTX_CONTESTED) { 656 owner = casuword(&umtx->u_owner, 657 UMTX_CONTESTED, id | UMTX_CONTESTED); 658 659 if (owner == UMTX_CONTESTED) 660 return (0); 661 662 /* The address was invalid. */ 663 if (owner == -1) 664 return (EFAULT); 665 666 /* If this failed the lock has changed, restart. */ 667 continue; 668 } 669 670 /* 671 * If we caught a signal, we have retried and now 672 * exit immediately. 673 */ 674 if (error != 0) 675 return (error); 676 677 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, 678 AUTO_SHARE, &uq->uq_key)) != 0) 679 return (error); 680 681 umtxq_lock(&uq->uq_key); 682 umtxq_busy(&uq->uq_key); 683 umtxq_insert(uq); 684 umtxq_unbusy(&uq->uq_key); 685 umtxq_unlock(&uq->uq_key); 686 687 /* 688 * Set the contested bit so that a release in user space 689 * knows to use the system call for unlock. If this fails 690 * either some one else has acquired the lock or it has been 691 * released. 692 */ 693 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED); 694 695 /* The address was invalid. */ 696 if (old == -1) { 697 umtxq_lock(&uq->uq_key); 698 umtxq_remove(uq); 699 umtxq_unlock(&uq->uq_key); 700 umtx_key_release(&uq->uq_key); 701 return (EFAULT); 702 } 703 704 /* 705 * We set the contested bit, sleep. Otherwise the lock changed 706 * and we need to retry or we lost a race to the thread 707 * unlocking the umtx. 708 */ 709 umtxq_lock(&uq->uq_key); 710 if (old == owner) 711 error = umtxq_sleep(uq, "umtx", timo); 712 umtxq_remove(uq); 713 umtxq_unlock(&uq->uq_key); 714 umtx_key_release(&uq->uq_key); 715 } 716 717 return (0); 718 } 719 720 /* 721 * Lock a umtx object. 722 */ 723 static int 724 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, 725 struct timespec *timeout) 726 { 727 struct timespec ts, ts2, ts3; 728 struct timeval tv; 729 int error; 730 731 if (timeout == NULL) { 732 error = _do_lock_umtx(td, umtx, id, 0); 733 /* Mutex locking is restarted if it is interrupted. */ 734 if (error == EINTR) 735 error = ERESTART; 736 } else { 737 getnanouptime(&ts); 738 timespecadd(&ts, timeout); 739 TIMESPEC_TO_TIMEVAL(&tv, timeout); 740 for (;;) { 741 error = _do_lock_umtx(td, umtx, id, tvtohz(&tv)); 742 if (error != ETIMEDOUT) 743 break; 744 getnanouptime(&ts2); 745 if (timespeccmp(&ts2, &ts, >=)) { 746 error = ETIMEDOUT; 747 break; 748 } 749 ts3 = ts; 750 timespecsub(&ts3, &ts2); 751 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 752 } 753 /* Timed-locking is not restarted. */ 754 if (error == ERESTART) 755 error = EINTR; 756 } 757 return (error); 758 } 759 760 /* 761 * Unlock a umtx object. 762 */ 763 static int 764 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id) 765 { 766 struct umtx_key key; 767 u_long owner; 768 u_long old; 769 int error; 770 int count; 771 772 /* 773 * Make sure we own this mtx. 774 */ 775 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner)); 776 if (owner == -1) 777 return (EFAULT); 778 779 if ((owner & ~UMTX_CONTESTED) != id) 780 return (EPERM); 781 782 /* This should be done in userland */ 783 if ((owner & UMTX_CONTESTED) == 0) { 784 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED); 785 if (old == -1) 786 return (EFAULT); 787 if (old == owner) 788 return (0); 789 owner = old; 790 } 791 792 /* We should only ever be in here for contested locks */ 793 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, 794 &key)) != 0) 795 return (error); 796 797 umtxq_lock(&key); 798 umtxq_busy(&key); 799 count = umtxq_count(&key); 800 umtxq_unlock(&key); 801 802 /* 803 * When unlocking the umtx, it must be marked as unowned if 804 * there is zero or one thread only waiting for it. 805 * Otherwise, it must be marked as contested. 806 */ 807 old = casuword(&umtx->u_owner, owner, 808 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED); 809 umtxq_lock(&key); 810 umtxq_signal(&key,1); 811 umtxq_unbusy(&key); 812 umtxq_unlock(&key); 813 umtx_key_release(&key); 814 if (old == -1) 815 return (EFAULT); 816 if (old != owner) 817 return (EINVAL); 818 return (0); 819 } 820 821 #ifdef COMPAT_FREEBSD32 822 823 /* 824 * Lock a umtx object. 825 */ 826 static int 827 _do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo) 828 { 829 struct umtx_q *uq; 830 uint32_t owner; 831 uint32_t old; 832 int error = 0; 833 834 uq = td->td_umtxq; 835 836 /* 837 * Care must be exercised when dealing with umtx structure. It 838 * can fault on any access. 839 */ 840 for (;;) { 841 /* 842 * Try the uncontested case. This should be done in userland. 843 */ 844 owner = casuword32(m, UMUTEX_UNOWNED, id); 845 846 /* The acquire succeeded. */ 847 if (owner == UMUTEX_UNOWNED) 848 return (0); 849 850 /* The address was invalid. */ 851 if (owner == -1) 852 return (EFAULT); 853 854 /* If no one owns it but it is contested try to acquire it. */ 855 if (owner == UMUTEX_CONTESTED) { 856 owner = casuword32(m, 857 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 858 if (owner == UMUTEX_CONTESTED) 859 return (0); 860 861 /* The address was invalid. */ 862 if (owner == -1) 863 return (EFAULT); 864 865 /* If this failed the lock has changed, restart. */ 866 continue; 867 } 868 869 /* 870 * If we caught a signal, we have retried and now 871 * exit immediately. 872 */ 873 if (error != 0) 874 return (error); 875 876 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, 877 AUTO_SHARE, &uq->uq_key)) != 0) 878 return (error); 879 880 umtxq_lock(&uq->uq_key); 881 umtxq_busy(&uq->uq_key); 882 umtxq_insert(uq); 883 umtxq_unbusy(&uq->uq_key); 884 umtxq_unlock(&uq->uq_key); 885 886 /* 887 * Set the contested bit so that a release in user space 888 * knows to use the system call for unlock. If this fails 889 * either some one else has acquired the lock or it has been 890 * released. 891 */ 892 old = casuword32(m, owner, owner | UMUTEX_CONTESTED); 893 894 /* The address was invalid. */ 895 if (old == -1) { 896 umtxq_lock(&uq->uq_key); 897 umtxq_remove(uq); 898 umtxq_unlock(&uq->uq_key); 899 umtx_key_release(&uq->uq_key); 900 return (EFAULT); 901 } 902 903 /* 904 * We set the contested bit, sleep. Otherwise the lock changed 905 * and we need to retry or we lost a race to the thread 906 * unlocking the umtx. 907 */ 908 umtxq_lock(&uq->uq_key); 909 if (old == owner) 910 error = umtxq_sleep(uq, "umtx", timo); 911 umtxq_remove(uq); 912 umtxq_unlock(&uq->uq_key); 913 umtx_key_release(&uq->uq_key); 914 } 915 916 return (0); 917 } 918 919 /* 920 * Lock a umtx object. 921 */ 922 static int 923 do_lock_umtx32(struct thread *td, void *m, uint32_t id, 924 struct timespec *timeout) 925 { 926 struct timespec ts, ts2, ts3; 927 struct timeval tv; 928 int error; 929 930 if (timeout == NULL) { 931 error = _do_lock_umtx32(td, m, id, 0); 932 /* Mutex locking is restarted if it is interrupted. */ 933 if (error == EINTR) 934 error = ERESTART; 935 } else { 936 getnanouptime(&ts); 937 timespecadd(&ts, timeout); 938 TIMESPEC_TO_TIMEVAL(&tv, timeout); 939 for (;;) { 940 error = _do_lock_umtx32(td, m, id, tvtohz(&tv)); 941 if (error != ETIMEDOUT) 942 break; 943 getnanouptime(&ts2); 944 if (timespeccmp(&ts2, &ts, >=)) { 945 error = ETIMEDOUT; 946 break; 947 } 948 ts3 = ts; 949 timespecsub(&ts3, &ts2); 950 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 951 } 952 /* Timed-locking is not restarted. */ 953 if (error == ERESTART) 954 error = EINTR; 955 } 956 return (error); 957 } 958 959 /* 960 * Unlock a umtx object. 961 */ 962 static int 963 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id) 964 { 965 struct umtx_key key; 966 uint32_t owner; 967 uint32_t old; 968 int error; 969 int count; 970 971 /* 972 * Make sure we own this mtx. 973 */ 974 owner = fuword32(m); 975 if (owner == -1) 976 return (EFAULT); 977 978 if ((owner & ~UMUTEX_CONTESTED) != id) 979 return (EPERM); 980 981 /* This should be done in userland */ 982 if ((owner & UMUTEX_CONTESTED) == 0) { 983 old = casuword32(m, owner, UMUTEX_UNOWNED); 984 if (old == -1) 985 return (EFAULT); 986 if (old == owner) 987 return (0); 988 owner = old; 989 } 990 991 /* We should only ever be in here for contested locks */ 992 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE, 993 &key)) != 0) 994 return (error); 995 996 umtxq_lock(&key); 997 umtxq_busy(&key); 998 count = umtxq_count(&key); 999 umtxq_unlock(&key); 1000 1001 /* 1002 * When unlocking the umtx, it must be marked as unowned if 1003 * there is zero or one thread only waiting for it. 1004 * Otherwise, it must be marked as contested. 1005 */ 1006 old = casuword32(m, owner, 1007 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 1008 umtxq_lock(&key); 1009 umtxq_signal(&key,1); 1010 umtxq_unbusy(&key); 1011 umtxq_unlock(&key); 1012 umtx_key_release(&key); 1013 if (old == -1) 1014 return (EFAULT); 1015 if (old != owner) 1016 return (EINVAL); 1017 return (0); 1018 } 1019 #endif 1020 1021 /* 1022 * Fetch and compare value, sleep on the address if value is not changed. 1023 */ 1024 static int 1025 do_wait(struct thread *td, void *addr, u_long id, 1026 struct timespec *timeout, int compat32, int is_private) 1027 { 1028 struct umtx_q *uq; 1029 struct timespec ts, ts2, ts3; 1030 struct timeval tv; 1031 u_long tmp; 1032 int error = 0; 1033 1034 uq = td->td_umtxq; 1035 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, 1036 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) 1037 return (error); 1038 1039 umtxq_lock(&uq->uq_key); 1040 umtxq_insert(uq); 1041 umtxq_unlock(&uq->uq_key); 1042 if (compat32 == 0) 1043 tmp = fuword(addr); 1044 else 1045 tmp = (unsigned int)fuword32(addr); 1046 if (tmp != id) { 1047 umtxq_lock(&uq->uq_key); 1048 umtxq_remove(uq); 1049 umtxq_unlock(&uq->uq_key); 1050 } else if (timeout == NULL) { 1051 umtxq_lock(&uq->uq_key); 1052 error = umtxq_sleep(uq, "uwait", 0); 1053 umtxq_remove(uq); 1054 umtxq_unlock(&uq->uq_key); 1055 } else { 1056 getnanouptime(&ts); 1057 timespecadd(&ts, timeout); 1058 TIMESPEC_TO_TIMEVAL(&tv, timeout); 1059 umtxq_lock(&uq->uq_key); 1060 for (;;) { 1061 error = umtxq_sleep(uq, "uwait", tvtohz(&tv)); 1062 if (!(uq->uq_flags & UQF_UMTXQ)) { 1063 error = 0; 1064 break; 1065 } 1066 if (error != ETIMEDOUT) 1067 break; 1068 umtxq_unlock(&uq->uq_key); 1069 getnanouptime(&ts2); 1070 if (timespeccmp(&ts2, &ts, >=)) { 1071 error = ETIMEDOUT; 1072 umtxq_lock(&uq->uq_key); 1073 break; 1074 } 1075 ts3 = ts; 1076 timespecsub(&ts3, &ts2); 1077 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 1078 umtxq_lock(&uq->uq_key); 1079 } 1080 umtxq_remove(uq); 1081 umtxq_unlock(&uq->uq_key); 1082 } 1083 umtx_key_release(&uq->uq_key); 1084 if (error == ERESTART) 1085 error = EINTR; 1086 return (error); 1087 } 1088 1089 /* 1090 * Wake up threads sleeping on the specified address. 1091 */ 1092 int 1093 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) 1094 { 1095 struct umtx_key key; 1096 int ret; 1097 1098 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, 1099 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) 1100 return (ret); 1101 umtxq_lock(&key); 1102 ret = umtxq_signal(&key, n_wake); 1103 umtxq_unlock(&key); 1104 umtx_key_release(&key); 1105 return (0); 1106 } 1107 1108 /* 1109 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1110 */ 1111 static int 1112 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1113 int mode) 1114 { 1115 struct umtx_q *uq; 1116 uint32_t owner, old, id; 1117 int error = 0; 1118 1119 id = td->td_tid; 1120 uq = td->td_umtxq; 1121 1122 /* 1123 * Care must be exercised when dealing with umtx structure. It 1124 * can fault on any access. 1125 */ 1126 for (;;) { 1127 owner = fuword32(__DEVOLATILE(void *, &m->m_owner)); 1128 if (mode == _UMUTEX_WAIT) { 1129 if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED) 1130 return (0); 1131 } else { 1132 /* 1133 * Try the uncontested case. This should be done in userland. 1134 */ 1135 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); 1136 1137 /* The acquire succeeded. */ 1138 if (owner == UMUTEX_UNOWNED) 1139 return (0); 1140 1141 /* The address was invalid. */ 1142 if (owner == -1) 1143 return (EFAULT); 1144 1145 /* If no one owns it but it is contested try to acquire it. */ 1146 if (owner == UMUTEX_CONTESTED) { 1147 owner = casuword32(&m->m_owner, 1148 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1149 1150 if (owner == UMUTEX_CONTESTED) 1151 return (0); 1152 1153 /* The address was invalid. */ 1154 if (owner == -1) 1155 return (EFAULT); 1156 1157 /* If this failed the lock has changed, restart. */ 1158 continue; 1159 } 1160 } 1161 1162 if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1163 (owner & ~UMUTEX_CONTESTED) == id) 1164 return (EDEADLK); 1165 1166 if (mode == _UMUTEX_TRY) 1167 return (EBUSY); 1168 1169 /* 1170 * If we caught a signal, we have retried and now 1171 * exit immediately. 1172 */ 1173 if (error != 0) 1174 return (error); 1175 1176 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, 1177 GET_SHARE(flags), &uq->uq_key)) != 0) 1178 return (error); 1179 1180 umtxq_lock(&uq->uq_key); 1181 umtxq_busy(&uq->uq_key); 1182 umtxq_insert(uq); 1183 umtxq_unlock(&uq->uq_key); 1184 1185 /* 1186 * Set the contested bit so that a release in user space 1187 * knows to use the system call for unlock. If this fails 1188 * either some one else has acquired the lock or it has been 1189 * released. 1190 */ 1191 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); 1192 1193 /* The address was invalid. */ 1194 if (old == -1) { 1195 umtxq_lock(&uq->uq_key); 1196 umtxq_remove(uq); 1197 umtxq_unbusy(&uq->uq_key); 1198 umtxq_unlock(&uq->uq_key); 1199 umtx_key_release(&uq->uq_key); 1200 return (EFAULT); 1201 } 1202 1203 /* 1204 * We set the contested bit, sleep. Otherwise the lock changed 1205 * and we need to retry or we lost a race to the thread 1206 * unlocking the umtx. 1207 */ 1208 umtxq_lock(&uq->uq_key); 1209 umtxq_unbusy(&uq->uq_key); 1210 if (old == owner) 1211 error = umtxq_sleep(uq, "umtxn", timo); 1212 umtxq_remove(uq); 1213 umtxq_unlock(&uq->uq_key); 1214 umtx_key_release(&uq->uq_key); 1215 } 1216 1217 return (0); 1218 } 1219 1220 /* 1221 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1222 */ 1223 /* 1224 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. 1225 */ 1226 static int 1227 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) 1228 { 1229 struct umtx_key key; 1230 uint32_t owner, old, id; 1231 int error; 1232 int count; 1233 1234 id = td->td_tid; 1235 /* 1236 * Make sure we own this mtx. 1237 */ 1238 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1239 if (owner == -1) 1240 return (EFAULT); 1241 1242 if ((owner & ~UMUTEX_CONTESTED) != id) 1243 return (EPERM); 1244 1245 if ((owner & UMUTEX_CONTESTED) == 0) { 1246 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); 1247 if (old == -1) 1248 return (EFAULT); 1249 if (old == owner) 1250 return (0); 1251 owner = old; 1252 } 1253 1254 /* We should only ever be in here for contested locks */ 1255 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1256 &key)) != 0) 1257 return (error); 1258 1259 umtxq_lock(&key); 1260 umtxq_busy(&key); 1261 count = umtxq_count(&key); 1262 umtxq_unlock(&key); 1263 1264 /* 1265 * When unlocking the umtx, it must be marked as unowned if 1266 * there is zero or one thread only waiting for it. 1267 * Otherwise, it must be marked as contested. 1268 */ 1269 old = casuword32(&m->m_owner, owner, 1270 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 1271 umtxq_lock(&key); 1272 umtxq_signal(&key,1); 1273 umtxq_unbusy(&key); 1274 umtxq_unlock(&key); 1275 umtx_key_release(&key); 1276 if (old == -1) 1277 return (EFAULT); 1278 if (old != owner) 1279 return (EINVAL); 1280 return (0); 1281 } 1282 1283 /* 1284 * Check if the mutex is available and wake up a waiter, 1285 * only for simple mutex. 1286 */ 1287 static int 1288 do_wake_umutex(struct thread *td, struct umutex *m) 1289 { 1290 struct umtx_key key; 1291 uint32_t owner; 1292 uint32_t flags; 1293 int error; 1294 int count; 1295 1296 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1297 if (owner == -1) 1298 return (EFAULT); 1299 1300 if ((owner & ~UMUTEX_CONTESTED) != 0) 1301 return (0); 1302 1303 flags = fuword32(&m->m_flags); 1304 1305 /* We should only ever be in here for contested locks */ 1306 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1307 &key)) != 0) 1308 return (error); 1309 1310 umtxq_lock(&key); 1311 umtxq_busy(&key); 1312 count = umtxq_count(&key); 1313 umtxq_unlock(&key); 1314 1315 if (count <= 1) 1316 owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED); 1317 1318 umtxq_lock(&key); 1319 if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) 1320 umtxq_signal(&key, 1); 1321 umtxq_unbusy(&key); 1322 umtxq_unlock(&key); 1323 umtx_key_release(&key); 1324 return (0); 1325 } 1326 1327 static inline struct umtx_pi * 1328 umtx_pi_alloc(int flags) 1329 { 1330 struct umtx_pi *pi; 1331 1332 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags); 1333 TAILQ_INIT(&pi->pi_blocked); 1334 atomic_add_int(&umtx_pi_allocated, 1); 1335 return (pi); 1336 } 1337 1338 static inline void 1339 umtx_pi_free(struct umtx_pi *pi) 1340 { 1341 uma_zfree(umtx_pi_zone, pi); 1342 atomic_add_int(&umtx_pi_allocated, -1); 1343 } 1344 1345 /* 1346 * Adjust the thread's position on a pi_state after its priority has been 1347 * changed. 1348 */ 1349 static int 1350 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) 1351 { 1352 struct umtx_q *uq, *uq1, *uq2; 1353 struct thread *td1; 1354 1355 mtx_assert(&umtx_lock, MA_OWNED); 1356 if (pi == NULL) 1357 return (0); 1358 1359 uq = td->td_umtxq; 1360 1361 /* 1362 * Check if the thread needs to be moved on the blocked chain. 1363 * It needs to be moved if either its priority is lower than 1364 * the previous thread or higher than the next thread. 1365 */ 1366 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq); 1367 uq2 = TAILQ_NEXT(uq, uq_lockq); 1368 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || 1369 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { 1370 /* 1371 * Remove thread from blocked chain and determine where 1372 * it should be moved to. 1373 */ 1374 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1375 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1376 td1 = uq1->uq_thread; 1377 MPASS(td1->td_proc->p_magic == P_MAGIC); 1378 if (UPRI(td1) > UPRI(td)) 1379 break; 1380 } 1381 1382 if (uq1 == NULL) 1383 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1384 else 1385 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1386 } 1387 return (1); 1388 } 1389 1390 /* 1391 * Propagate priority when a thread is blocked on POSIX 1392 * PI mutex. 1393 */ 1394 static void 1395 umtx_propagate_priority(struct thread *td) 1396 { 1397 struct umtx_q *uq; 1398 struct umtx_pi *pi; 1399 int pri; 1400 1401 mtx_assert(&umtx_lock, MA_OWNED); 1402 pri = UPRI(td); 1403 uq = td->td_umtxq; 1404 pi = uq->uq_pi_blocked; 1405 if (pi == NULL) 1406 return; 1407 1408 for (;;) { 1409 td = pi->pi_owner; 1410 if (td == NULL || td == curthread) 1411 return; 1412 1413 MPASS(td->td_proc != NULL); 1414 MPASS(td->td_proc->p_magic == P_MAGIC); 1415 1416 thread_lock(td); 1417 if (td->td_lend_user_pri > pri) 1418 sched_lend_user_prio(td, pri); 1419 else { 1420 thread_unlock(td); 1421 break; 1422 } 1423 thread_unlock(td); 1424 1425 /* 1426 * Pick up the lock that td is blocked on. 1427 */ 1428 uq = td->td_umtxq; 1429 pi = uq->uq_pi_blocked; 1430 /* Resort td on the list if needed. */ 1431 if (!umtx_pi_adjust_thread(pi, td)) 1432 break; 1433 } 1434 } 1435 1436 /* 1437 * Unpropagate priority for a PI mutex when a thread blocked on 1438 * it is interrupted by signal or resumed by others. 1439 */ 1440 static void 1441 umtx_unpropagate_priority(struct umtx_pi *pi) 1442 { 1443 struct umtx_q *uq, *uq_owner; 1444 struct umtx_pi *pi2; 1445 int pri, oldpri; 1446 1447 mtx_assert(&umtx_lock, MA_OWNED); 1448 1449 while (pi != NULL && pi->pi_owner != NULL) { 1450 pri = PRI_MAX; 1451 uq_owner = pi->pi_owner->td_umtxq; 1452 1453 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { 1454 uq = TAILQ_FIRST(&pi2->pi_blocked); 1455 if (uq != NULL) { 1456 if (pri > UPRI(uq->uq_thread)) 1457 pri = UPRI(uq->uq_thread); 1458 } 1459 } 1460 1461 if (pri > uq_owner->uq_inherited_pri) 1462 pri = uq_owner->uq_inherited_pri; 1463 thread_lock(pi->pi_owner); 1464 oldpri = pi->pi_owner->td_user_pri; 1465 sched_unlend_user_prio(pi->pi_owner, pri); 1466 thread_unlock(pi->pi_owner); 1467 if (uq_owner->uq_pi_blocked != NULL) 1468 umtx_pi_adjust_locked(pi->pi_owner, oldpri); 1469 pi = uq_owner->uq_pi_blocked; 1470 } 1471 } 1472 1473 /* 1474 * Insert a PI mutex into owned list. 1475 */ 1476 static void 1477 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) 1478 { 1479 struct umtx_q *uq_owner; 1480 1481 uq_owner = owner->td_umtxq; 1482 mtx_assert(&umtx_lock, MA_OWNED); 1483 if (pi->pi_owner != NULL) 1484 panic("pi_ower != NULL"); 1485 pi->pi_owner = owner; 1486 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); 1487 } 1488 1489 /* 1490 * Claim ownership of a PI mutex. 1491 */ 1492 static int 1493 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) 1494 { 1495 struct umtx_q *uq, *uq_owner; 1496 1497 uq_owner = owner->td_umtxq; 1498 mtx_lock_spin(&umtx_lock); 1499 if (pi->pi_owner == owner) { 1500 mtx_unlock_spin(&umtx_lock); 1501 return (0); 1502 } 1503 1504 if (pi->pi_owner != NULL) { 1505 /* 1506 * userland may have already messed the mutex, sigh. 1507 */ 1508 mtx_unlock_spin(&umtx_lock); 1509 return (EPERM); 1510 } 1511 umtx_pi_setowner(pi, owner); 1512 uq = TAILQ_FIRST(&pi->pi_blocked); 1513 if (uq != NULL) { 1514 int pri; 1515 1516 pri = UPRI(uq->uq_thread); 1517 thread_lock(owner); 1518 if (pri < UPRI(owner)) 1519 sched_lend_user_prio(owner, pri); 1520 thread_unlock(owner); 1521 } 1522 mtx_unlock_spin(&umtx_lock); 1523 return (0); 1524 } 1525 1526 static void 1527 umtx_pi_adjust_locked(struct thread *td, u_char oldpri) 1528 { 1529 struct umtx_q *uq; 1530 struct umtx_pi *pi; 1531 1532 uq = td->td_umtxq; 1533 /* 1534 * Pick up the lock that td is blocked on. 1535 */ 1536 pi = uq->uq_pi_blocked; 1537 MPASS(pi != NULL); 1538 1539 /* Resort the turnstile on the list. */ 1540 if (!umtx_pi_adjust_thread(pi, td)) 1541 return; 1542 1543 /* 1544 * If our priority was lowered and we are at the head of the 1545 * turnstile, then propagate our new priority up the chain. 1546 */ 1547 if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri) 1548 umtx_propagate_priority(td); 1549 } 1550 1551 /* 1552 * Adjust a thread's order position in its blocked PI mutex, 1553 * this may result new priority propagating process. 1554 */ 1555 void 1556 umtx_pi_adjust(struct thread *td, u_char oldpri) 1557 { 1558 struct umtx_q *uq; 1559 struct umtx_pi *pi; 1560 1561 uq = td->td_umtxq; 1562 mtx_lock_spin(&umtx_lock); 1563 /* 1564 * Pick up the lock that td is blocked on. 1565 */ 1566 pi = uq->uq_pi_blocked; 1567 if (pi != NULL) 1568 umtx_pi_adjust_locked(td, oldpri); 1569 mtx_unlock_spin(&umtx_lock); 1570 } 1571 1572 /* 1573 * Sleep on a PI mutex. 1574 */ 1575 static int 1576 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, 1577 uint32_t owner, const char *wmesg, int timo) 1578 { 1579 struct umtxq_chain *uc; 1580 struct thread *td, *td1; 1581 struct umtx_q *uq1; 1582 int pri; 1583 int error = 0; 1584 1585 td = uq->uq_thread; 1586 KASSERT(td == curthread, ("inconsistent uq_thread")); 1587 uc = umtxq_getchain(&uq->uq_key); 1588 UMTXQ_LOCKED_ASSERT(uc); 1589 UMTXQ_BUSY_ASSERT(uc); 1590 umtxq_insert(uq); 1591 mtx_lock_spin(&umtx_lock); 1592 if (pi->pi_owner == NULL) { 1593 mtx_unlock_spin(&umtx_lock); 1594 /* XXX Only look up thread in current process. */ 1595 td1 = tdfind(owner, curproc->p_pid); 1596 mtx_lock_spin(&umtx_lock); 1597 if (td1 != NULL) { 1598 if (pi->pi_owner == NULL) 1599 umtx_pi_setowner(pi, td1); 1600 PROC_UNLOCK(td1->td_proc); 1601 } 1602 } 1603 1604 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1605 pri = UPRI(uq1->uq_thread); 1606 if (pri > UPRI(td)) 1607 break; 1608 } 1609 1610 if (uq1 != NULL) 1611 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1612 else 1613 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1614 1615 uq->uq_pi_blocked = pi; 1616 thread_lock(td); 1617 td->td_flags |= TDF_UPIBLOCKED; 1618 thread_unlock(td); 1619 umtx_propagate_priority(td); 1620 mtx_unlock_spin(&umtx_lock); 1621 umtxq_unbusy(&uq->uq_key); 1622 1623 if (uq->uq_flags & UQF_UMTXQ) { 1624 error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); 1625 if (error == EWOULDBLOCK) 1626 error = ETIMEDOUT; 1627 if (uq->uq_flags & UQF_UMTXQ) { 1628 umtxq_remove(uq); 1629 } 1630 } 1631 mtx_lock_spin(&umtx_lock); 1632 uq->uq_pi_blocked = NULL; 1633 thread_lock(td); 1634 td->td_flags &= ~TDF_UPIBLOCKED; 1635 thread_unlock(td); 1636 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1637 umtx_unpropagate_priority(pi); 1638 mtx_unlock_spin(&umtx_lock); 1639 umtxq_unlock(&uq->uq_key); 1640 1641 return (error); 1642 } 1643 1644 /* 1645 * Add reference count for a PI mutex. 1646 */ 1647 static void 1648 umtx_pi_ref(struct umtx_pi *pi) 1649 { 1650 struct umtxq_chain *uc; 1651 1652 uc = umtxq_getchain(&pi->pi_key); 1653 UMTXQ_LOCKED_ASSERT(uc); 1654 pi->pi_refcount++; 1655 } 1656 1657 /* 1658 * Decrease reference count for a PI mutex, if the counter 1659 * is decreased to zero, its memory space is freed. 1660 */ 1661 static void 1662 umtx_pi_unref(struct umtx_pi *pi) 1663 { 1664 struct umtxq_chain *uc; 1665 1666 uc = umtxq_getchain(&pi->pi_key); 1667 UMTXQ_LOCKED_ASSERT(uc); 1668 KASSERT(pi->pi_refcount > 0, ("invalid reference count")); 1669 if (--pi->pi_refcount == 0) { 1670 mtx_lock_spin(&umtx_lock); 1671 if (pi->pi_owner != NULL) { 1672 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, 1673 pi, pi_link); 1674 pi->pi_owner = NULL; 1675 } 1676 KASSERT(TAILQ_EMPTY(&pi->pi_blocked), 1677 ("blocked queue not empty")); 1678 mtx_unlock_spin(&umtx_lock); 1679 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); 1680 umtx_pi_free(pi); 1681 } 1682 } 1683 1684 /* 1685 * Find a PI mutex in hash table. 1686 */ 1687 static struct umtx_pi * 1688 umtx_pi_lookup(struct umtx_key *key) 1689 { 1690 struct umtxq_chain *uc; 1691 struct umtx_pi *pi; 1692 1693 uc = umtxq_getchain(key); 1694 UMTXQ_LOCKED_ASSERT(uc); 1695 1696 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { 1697 if (umtx_key_match(&pi->pi_key, key)) { 1698 return (pi); 1699 } 1700 } 1701 return (NULL); 1702 } 1703 1704 /* 1705 * Insert a PI mutex into hash table. 1706 */ 1707 static inline void 1708 umtx_pi_insert(struct umtx_pi *pi) 1709 { 1710 struct umtxq_chain *uc; 1711 1712 uc = umtxq_getchain(&pi->pi_key); 1713 UMTXQ_LOCKED_ASSERT(uc); 1714 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); 1715 } 1716 1717 /* 1718 * Lock a PI mutex. 1719 */ 1720 static int 1721 _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1722 int try) 1723 { 1724 struct umtx_q *uq; 1725 struct umtx_pi *pi, *new_pi; 1726 uint32_t id, owner, old; 1727 int error; 1728 1729 id = td->td_tid; 1730 uq = td->td_umtxq; 1731 1732 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), 1733 &uq->uq_key)) != 0) 1734 return (error); 1735 umtxq_lock(&uq->uq_key); 1736 pi = umtx_pi_lookup(&uq->uq_key); 1737 if (pi == NULL) { 1738 new_pi = umtx_pi_alloc(M_NOWAIT); 1739 if (new_pi == NULL) { 1740 umtxq_unlock(&uq->uq_key); 1741 new_pi = umtx_pi_alloc(M_WAITOK); 1742 umtxq_lock(&uq->uq_key); 1743 pi = umtx_pi_lookup(&uq->uq_key); 1744 if (pi != NULL) { 1745 umtx_pi_free(new_pi); 1746 new_pi = NULL; 1747 } 1748 } 1749 if (new_pi != NULL) { 1750 new_pi->pi_key = uq->uq_key; 1751 umtx_pi_insert(new_pi); 1752 pi = new_pi; 1753 } 1754 } 1755 umtx_pi_ref(pi); 1756 umtxq_unlock(&uq->uq_key); 1757 1758 /* 1759 * Care must be exercised when dealing with umtx structure. It 1760 * can fault on any access. 1761 */ 1762 for (;;) { 1763 /* 1764 * Try the uncontested case. This should be done in userland. 1765 */ 1766 owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); 1767 1768 /* The acquire succeeded. */ 1769 if (owner == UMUTEX_UNOWNED) { 1770 error = 0; 1771 break; 1772 } 1773 1774 /* The address was invalid. */ 1775 if (owner == -1) { 1776 error = EFAULT; 1777 break; 1778 } 1779 1780 /* If no one owns it but it is contested try to acquire it. */ 1781 if (owner == UMUTEX_CONTESTED) { 1782 owner = casuword32(&m->m_owner, 1783 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1784 1785 if (owner == UMUTEX_CONTESTED) { 1786 umtxq_lock(&uq->uq_key); 1787 umtxq_busy(&uq->uq_key); 1788 error = umtx_pi_claim(pi, td); 1789 umtxq_unbusy(&uq->uq_key); 1790 umtxq_unlock(&uq->uq_key); 1791 break; 1792 } 1793 1794 /* The address was invalid. */ 1795 if (owner == -1) { 1796 error = EFAULT; 1797 break; 1798 } 1799 1800 /* If this failed the lock has changed, restart. */ 1801 continue; 1802 } 1803 1804 if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1805 (owner & ~UMUTEX_CONTESTED) == id) { 1806 error = EDEADLK; 1807 break; 1808 } 1809 1810 if (try != 0) { 1811 error = EBUSY; 1812 break; 1813 } 1814 1815 /* 1816 * If we caught a signal, we have retried and now 1817 * exit immediately. 1818 */ 1819 if (error != 0) 1820 break; 1821 1822 umtxq_lock(&uq->uq_key); 1823 umtxq_busy(&uq->uq_key); 1824 umtxq_unlock(&uq->uq_key); 1825 1826 /* 1827 * Set the contested bit so that a release in user space 1828 * knows to use the system call for unlock. If this fails 1829 * either some one else has acquired the lock or it has been 1830 * released. 1831 */ 1832 old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); 1833 1834 /* The address was invalid. */ 1835 if (old == -1) { 1836 umtxq_lock(&uq->uq_key); 1837 umtxq_unbusy(&uq->uq_key); 1838 umtxq_unlock(&uq->uq_key); 1839 error = EFAULT; 1840 break; 1841 } 1842 1843 umtxq_lock(&uq->uq_key); 1844 /* 1845 * We set the contested bit, sleep. Otherwise the lock changed 1846 * and we need to retry or we lost a race to the thread 1847 * unlocking the umtx. 1848 */ 1849 if (old == owner) 1850 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, 1851 "umtxpi", timo); 1852 else { 1853 umtxq_unbusy(&uq->uq_key); 1854 umtxq_unlock(&uq->uq_key); 1855 } 1856 } 1857 1858 umtxq_lock(&uq->uq_key); 1859 umtx_pi_unref(pi); 1860 umtxq_unlock(&uq->uq_key); 1861 1862 umtx_key_release(&uq->uq_key); 1863 return (error); 1864 } 1865 1866 /* 1867 * Unlock a PI mutex. 1868 */ 1869 static int 1870 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) 1871 { 1872 struct umtx_key key; 1873 struct umtx_q *uq_first, *uq_first2, *uq_me; 1874 struct umtx_pi *pi, *pi2; 1875 uint32_t owner, old, id; 1876 int error; 1877 int count; 1878 int pri; 1879 1880 id = td->td_tid; 1881 /* 1882 * Make sure we own this mtx. 1883 */ 1884 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1885 if (owner == -1) 1886 return (EFAULT); 1887 1888 if ((owner & ~UMUTEX_CONTESTED) != id) 1889 return (EPERM); 1890 1891 /* This should be done in userland */ 1892 if ((owner & UMUTEX_CONTESTED) == 0) { 1893 old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); 1894 if (old == -1) 1895 return (EFAULT); 1896 if (old == owner) 1897 return (0); 1898 owner = old; 1899 } 1900 1901 /* We should only ever be in here for contested locks */ 1902 if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), 1903 &key)) != 0) 1904 return (error); 1905 1906 umtxq_lock(&key); 1907 umtxq_busy(&key); 1908 count = umtxq_count_pi(&key, &uq_first); 1909 if (uq_first != NULL) { 1910 mtx_lock_spin(&umtx_lock); 1911 pi = uq_first->uq_pi_blocked; 1912 KASSERT(pi != NULL, ("pi == NULL?")); 1913 if (pi->pi_owner != curthread) { 1914 mtx_unlock_spin(&umtx_lock); 1915 umtxq_unbusy(&key); 1916 umtxq_unlock(&key); 1917 umtx_key_release(&key); 1918 /* userland messed the mutex */ 1919 return (EPERM); 1920 } 1921 uq_me = curthread->td_umtxq; 1922 pi->pi_owner = NULL; 1923 TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link); 1924 /* get highest priority thread which is still sleeping. */ 1925 uq_first = TAILQ_FIRST(&pi->pi_blocked); 1926 while (uq_first != NULL && 1927 (uq_first->uq_flags & UQF_UMTXQ) == 0) { 1928 uq_first = TAILQ_NEXT(uq_first, uq_lockq); 1929 } 1930 pri = PRI_MAX; 1931 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { 1932 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); 1933 if (uq_first2 != NULL) { 1934 if (pri > UPRI(uq_first2->uq_thread)) 1935 pri = UPRI(uq_first2->uq_thread); 1936 } 1937 } 1938 thread_lock(curthread); 1939 sched_unlend_user_prio(curthread, pri); 1940 thread_unlock(curthread); 1941 mtx_unlock_spin(&umtx_lock); 1942 if (uq_first) 1943 umtxq_signal_thread(uq_first); 1944 } 1945 umtxq_unlock(&key); 1946 1947 /* 1948 * When unlocking the umtx, it must be marked as unowned if 1949 * there is zero or one thread only waiting for it. 1950 * Otherwise, it must be marked as contested. 1951 */ 1952 old = casuword32(&m->m_owner, owner, 1953 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 1954 1955 umtxq_lock(&key); 1956 umtxq_unbusy(&key); 1957 umtxq_unlock(&key); 1958 umtx_key_release(&key); 1959 if (old == -1) 1960 return (EFAULT); 1961 if (old != owner) 1962 return (EINVAL); 1963 return (0); 1964 } 1965 1966 /* 1967 * Lock a PP mutex. 1968 */ 1969 static int 1970 _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1971 int try) 1972 { 1973 struct umtx_q *uq, *uq2; 1974 struct umtx_pi *pi; 1975 uint32_t ceiling; 1976 uint32_t owner, id; 1977 int error, pri, old_inherited_pri, su; 1978 1979 id = td->td_tid; 1980 uq = td->td_umtxq; 1981 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 1982 &uq->uq_key)) != 0) 1983 return (error); 1984 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 1985 for (;;) { 1986 old_inherited_pri = uq->uq_inherited_pri; 1987 umtxq_lock(&uq->uq_key); 1988 umtxq_busy(&uq->uq_key); 1989 umtxq_unlock(&uq->uq_key); 1990 1991 ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]); 1992 if (ceiling > RTP_PRIO_MAX) { 1993 error = EINVAL; 1994 goto out; 1995 } 1996 1997 mtx_lock_spin(&umtx_lock); 1998 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { 1999 mtx_unlock_spin(&umtx_lock); 2000 error = EINVAL; 2001 goto out; 2002 } 2003 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { 2004 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; 2005 thread_lock(td); 2006 if (uq->uq_inherited_pri < UPRI(td)) 2007 sched_lend_user_prio(td, uq->uq_inherited_pri); 2008 thread_unlock(td); 2009 } 2010 mtx_unlock_spin(&umtx_lock); 2011 2012 owner = casuword32(&m->m_owner, 2013 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 2014 2015 if (owner == UMUTEX_CONTESTED) { 2016 error = 0; 2017 break; 2018 } 2019 2020 /* The address was invalid. */ 2021 if (owner == -1) { 2022 error = EFAULT; 2023 break; 2024 } 2025 2026 if ((flags & UMUTEX_ERROR_CHECK) != 0 && 2027 (owner & ~UMUTEX_CONTESTED) == id) { 2028 error = EDEADLK; 2029 break; 2030 } 2031 2032 if (try != 0) { 2033 error = EBUSY; 2034 break; 2035 } 2036 2037 /* 2038 * If we caught a signal, we have retried and now 2039 * exit immediately. 2040 */ 2041 if (error != 0) 2042 break; 2043 2044 umtxq_lock(&uq->uq_key); 2045 umtxq_insert(uq); 2046 umtxq_unbusy(&uq->uq_key); 2047 error = umtxq_sleep(uq, "umtxpp", timo); 2048 umtxq_remove(uq); 2049 umtxq_unlock(&uq->uq_key); 2050 2051 mtx_lock_spin(&umtx_lock); 2052 uq->uq_inherited_pri = old_inherited_pri; 2053 pri = PRI_MAX; 2054 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2055 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2056 if (uq2 != NULL) { 2057 if (pri > UPRI(uq2->uq_thread)) 2058 pri = UPRI(uq2->uq_thread); 2059 } 2060 } 2061 if (pri > uq->uq_inherited_pri) 2062 pri = uq->uq_inherited_pri; 2063 thread_lock(td); 2064 sched_unlend_user_prio(td, pri); 2065 thread_unlock(td); 2066 mtx_unlock_spin(&umtx_lock); 2067 } 2068 2069 if (error != 0) { 2070 mtx_lock_spin(&umtx_lock); 2071 uq->uq_inherited_pri = old_inherited_pri; 2072 pri = PRI_MAX; 2073 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2074 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2075 if (uq2 != NULL) { 2076 if (pri > UPRI(uq2->uq_thread)) 2077 pri = UPRI(uq2->uq_thread); 2078 } 2079 } 2080 if (pri > uq->uq_inherited_pri) 2081 pri = uq->uq_inherited_pri; 2082 thread_lock(td); 2083 sched_unlend_user_prio(td, pri); 2084 thread_unlock(td); 2085 mtx_unlock_spin(&umtx_lock); 2086 } 2087 2088 out: 2089 umtxq_lock(&uq->uq_key); 2090 umtxq_unbusy(&uq->uq_key); 2091 umtxq_unlock(&uq->uq_key); 2092 umtx_key_release(&uq->uq_key); 2093 return (error); 2094 } 2095 2096 /* 2097 * Unlock a PP mutex. 2098 */ 2099 static int 2100 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) 2101 { 2102 struct umtx_key key; 2103 struct umtx_q *uq, *uq2; 2104 struct umtx_pi *pi; 2105 uint32_t owner, id; 2106 uint32_t rceiling; 2107 int error, pri, new_inherited_pri, su; 2108 2109 id = td->td_tid; 2110 uq = td->td_umtxq; 2111 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 2112 2113 /* 2114 * Make sure we own this mtx. 2115 */ 2116 owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 2117 if (owner == -1) 2118 return (EFAULT); 2119 2120 if ((owner & ~UMUTEX_CONTESTED) != id) 2121 return (EPERM); 2122 2123 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); 2124 if (error != 0) 2125 return (error); 2126 2127 if (rceiling == -1) 2128 new_inherited_pri = PRI_MAX; 2129 else { 2130 rceiling = RTP_PRIO_MAX - rceiling; 2131 if (rceiling > RTP_PRIO_MAX) 2132 return (EINVAL); 2133 new_inherited_pri = PRI_MIN_REALTIME + rceiling; 2134 } 2135 2136 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 2137 &key)) != 0) 2138 return (error); 2139 umtxq_lock(&key); 2140 umtxq_busy(&key); 2141 umtxq_unlock(&key); 2142 /* 2143 * For priority protected mutex, always set unlocked state 2144 * to UMUTEX_CONTESTED, so that userland always enters kernel 2145 * to lock the mutex, it is necessary because thread priority 2146 * has to be adjusted for such mutex. 2147 */ 2148 error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner), 2149 UMUTEX_CONTESTED); 2150 2151 umtxq_lock(&key); 2152 if (error == 0) 2153 umtxq_signal(&key, 1); 2154 umtxq_unbusy(&key); 2155 umtxq_unlock(&key); 2156 2157 if (error == -1) 2158 error = EFAULT; 2159 else { 2160 mtx_lock_spin(&umtx_lock); 2161 if (su != 0) 2162 uq->uq_inherited_pri = new_inherited_pri; 2163 pri = PRI_MAX; 2164 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2165 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2166 if (uq2 != NULL) { 2167 if (pri > UPRI(uq2->uq_thread)) 2168 pri = UPRI(uq2->uq_thread); 2169 } 2170 } 2171 if (pri > uq->uq_inherited_pri) 2172 pri = uq->uq_inherited_pri; 2173 thread_lock(td); 2174 sched_unlend_user_prio(td, pri); 2175 thread_unlock(td); 2176 mtx_unlock_spin(&umtx_lock); 2177 } 2178 umtx_key_release(&key); 2179 return (error); 2180 } 2181 2182 static int 2183 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, 2184 uint32_t *old_ceiling) 2185 { 2186 struct umtx_q *uq; 2187 uint32_t save_ceiling; 2188 uint32_t owner, id; 2189 uint32_t flags; 2190 int error; 2191 2192 flags = fuword32(&m->m_flags); 2193 if ((flags & UMUTEX_PRIO_PROTECT) == 0) 2194 return (EINVAL); 2195 if (ceiling > RTP_PRIO_MAX) 2196 return (EINVAL); 2197 id = td->td_tid; 2198 uq = td->td_umtxq; 2199 if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 2200 &uq->uq_key)) != 0) 2201 return (error); 2202 for (;;) { 2203 umtxq_lock(&uq->uq_key); 2204 umtxq_busy(&uq->uq_key); 2205 umtxq_unlock(&uq->uq_key); 2206 2207 save_ceiling = fuword32(&m->m_ceilings[0]); 2208 2209 owner = casuword32(&m->m_owner, 2210 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 2211 2212 if (owner == UMUTEX_CONTESTED) { 2213 suword32(&m->m_ceilings[0], ceiling); 2214 suword32(__DEVOLATILE(uint32_t *, &m->m_owner), 2215 UMUTEX_CONTESTED); 2216 error = 0; 2217 break; 2218 } 2219 2220 /* The address was invalid. */ 2221 if (owner == -1) { 2222 error = EFAULT; 2223 break; 2224 } 2225 2226 if ((owner & ~UMUTEX_CONTESTED) == id) { 2227 suword32(&m->m_ceilings[0], ceiling); 2228 error = 0; 2229 break; 2230 } 2231 2232 /* 2233 * If we caught a signal, we have retried and now 2234 * exit immediately. 2235 */ 2236 if (error != 0) 2237 break; 2238 2239 /* 2240 * We set the contested bit, sleep. Otherwise the lock changed 2241 * and we need to retry or we lost a race to the thread 2242 * unlocking the umtx. 2243 */ 2244 umtxq_lock(&uq->uq_key); 2245 umtxq_insert(uq); 2246 umtxq_unbusy(&uq->uq_key); 2247 error = umtxq_sleep(uq, "umtxpp", 0); 2248 umtxq_remove(uq); 2249 umtxq_unlock(&uq->uq_key); 2250 } 2251 umtxq_lock(&uq->uq_key); 2252 if (error == 0) 2253 umtxq_signal(&uq->uq_key, INT_MAX); 2254 umtxq_unbusy(&uq->uq_key); 2255 umtxq_unlock(&uq->uq_key); 2256 umtx_key_release(&uq->uq_key); 2257 if (error == 0 && old_ceiling != NULL) 2258 suword32(old_ceiling, save_ceiling); 2259 return (error); 2260 } 2261 2262 static int 2263 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo, 2264 int mode) 2265 { 2266 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2267 case 0: 2268 return (_do_lock_normal(td, m, flags, timo, mode)); 2269 case UMUTEX_PRIO_INHERIT: 2270 return (_do_lock_pi(td, m, flags, timo, mode)); 2271 case UMUTEX_PRIO_PROTECT: 2272 return (_do_lock_pp(td, m, flags, timo, mode)); 2273 } 2274 return (EINVAL); 2275 } 2276 2277 /* 2278 * Lock a userland POSIX mutex. 2279 */ 2280 static int 2281 do_lock_umutex(struct thread *td, struct umutex *m, 2282 struct timespec *timeout, int mode) 2283 { 2284 struct timespec ts, ts2, ts3; 2285 struct timeval tv; 2286 uint32_t flags; 2287 int error; 2288 2289 flags = fuword32(&m->m_flags); 2290 if (flags == -1) 2291 return (EFAULT); 2292 2293 if (timeout == NULL) { 2294 error = _do_lock_umutex(td, m, flags, 0, mode); 2295 /* Mutex locking is restarted if it is interrupted. */ 2296 if (error == EINTR && mode != _UMUTEX_WAIT) 2297 error = ERESTART; 2298 } else { 2299 getnanouptime(&ts); 2300 timespecadd(&ts, timeout); 2301 TIMESPEC_TO_TIMEVAL(&tv, timeout); 2302 for (;;) { 2303 error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode); 2304 if (error != ETIMEDOUT) 2305 break; 2306 getnanouptime(&ts2); 2307 if (timespeccmp(&ts2, &ts, >=)) { 2308 error = ETIMEDOUT; 2309 break; 2310 } 2311 ts3 = ts; 2312 timespecsub(&ts3, &ts2); 2313 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 2314 } 2315 /* Timed-locking is not restarted. */ 2316 if (error == ERESTART) 2317 error = EINTR; 2318 } 2319 return (error); 2320 } 2321 2322 /* 2323 * Unlock a userland POSIX mutex. 2324 */ 2325 static int 2326 do_unlock_umutex(struct thread *td, struct umutex *m) 2327 { 2328 uint32_t flags; 2329 2330 flags = fuword32(&m->m_flags); 2331 if (flags == -1) 2332 return (EFAULT); 2333 2334 switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2335 case 0: 2336 return (do_unlock_normal(td, m, flags)); 2337 case UMUTEX_PRIO_INHERIT: 2338 return (do_unlock_pi(td, m, flags)); 2339 case UMUTEX_PRIO_PROTECT: 2340 return (do_unlock_pp(td, m, flags)); 2341 } 2342 2343 return (EINVAL); 2344 } 2345 2346 static int 2347 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, 2348 struct timespec *timeout, u_long wflags) 2349 { 2350 struct umtx_q *uq; 2351 struct timeval tv; 2352 struct timespec cts, ets, tts; 2353 uint32_t flags; 2354 int error; 2355 2356 uq = td->td_umtxq; 2357 flags = fuword32(&cv->c_flags); 2358 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); 2359 if (error != 0) 2360 return (error); 2361 umtxq_lock(&uq->uq_key); 2362 umtxq_busy(&uq->uq_key); 2363 umtxq_insert(uq); 2364 umtxq_unlock(&uq->uq_key); 2365 2366 /* 2367 * The magic thing is we should set c_has_waiters to 1 before 2368 * releasing user mutex. 2369 */ 2370 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1); 2371 2372 umtxq_lock(&uq->uq_key); 2373 umtxq_unbusy(&uq->uq_key); 2374 umtxq_unlock(&uq->uq_key); 2375 2376 error = do_unlock_umutex(td, m); 2377 2378 umtxq_lock(&uq->uq_key); 2379 if (error == 0) { 2380 if ((wflags & UMTX_CHECK_UNPARKING) && 2381 (td->td_pflags & TDP_WAKEUP)) { 2382 td->td_pflags &= ~TDP_WAKEUP; 2383 error = EINTR; 2384 } else if (timeout == NULL) { 2385 error = umtxq_sleep(uq, "ucond", 0); 2386 } else { 2387 getnanouptime(&ets); 2388 timespecadd(&ets, timeout); 2389 TIMESPEC_TO_TIMEVAL(&tv, timeout); 2390 for (;;) { 2391 error = umtxq_sleep(uq, "ucond", tvtohz(&tv)); 2392 if (error != ETIMEDOUT) 2393 break; 2394 getnanouptime(&cts); 2395 if (timespeccmp(&cts, &ets, >=)) { 2396 error = ETIMEDOUT; 2397 break; 2398 } 2399 tts = ets; 2400 timespecsub(&tts, &cts); 2401 TIMESPEC_TO_TIMEVAL(&tv, &tts); 2402 } 2403 } 2404 } 2405 2406 if ((uq->uq_flags & UQF_UMTXQ) == 0) 2407 error = 0; 2408 else { 2409 umtxq_remove(uq); 2410 if (error == ERESTART) 2411 error = EINTR; 2412 } 2413 2414 umtxq_unlock(&uq->uq_key); 2415 umtx_key_release(&uq->uq_key); 2416 return (error); 2417 } 2418 2419 /* 2420 * Signal a userland condition variable. 2421 */ 2422 static int 2423 do_cv_signal(struct thread *td, struct ucond *cv) 2424 { 2425 struct umtx_key key; 2426 int error, cnt, nwake; 2427 uint32_t flags; 2428 2429 flags = fuword32(&cv->c_flags); 2430 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2431 return (error); 2432 umtxq_lock(&key); 2433 umtxq_busy(&key); 2434 cnt = umtxq_count(&key); 2435 nwake = umtxq_signal(&key, 1); 2436 if (cnt <= nwake) { 2437 umtxq_unlock(&key); 2438 error = suword32( 2439 __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); 2440 umtxq_lock(&key); 2441 } 2442 umtxq_unbusy(&key); 2443 umtxq_unlock(&key); 2444 umtx_key_release(&key); 2445 return (error); 2446 } 2447 2448 static int 2449 do_cv_broadcast(struct thread *td, struct ucond *cv) 2450 { 2451 struct umtx_key key; 2452 int error; 2453 uint32_t flags; 2454 2455 flags = fuword32(&cv->c_flags); 2456 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2457 return (error); 2458 2459 umtxq_lock(&key); 2460 umtxq_busy(&key); 2461 umtxq_signal(&key, INT_MAX); 2462 umtxq_unlock(&key); 2463 2464 error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); 2465 2466 umtxq_lock(&key); 2467 umtxq_unbusy(&key); 2468 umtxq_unlock(&key); 2469 2470 umtx_key_release(&key); 2471 return (error); 2472 } 2473 2474 static int 2475 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, int timo) 2476 { 2477 struct umtx_q *uq; 2478 uint32_t flags, wrflags; 2479 int32_t state, oldstate; 2480 int32_t blocked_readers; 2481 int error; 2482 2483 uq = td->td_umtxq; 2484 flags = fuword32(&rwlock->rw_flags); 2485 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2486 if (error != 0) 2487 return (error); 2488 2489 wrflags = URWLOCK_WRITE_OWNER; 2490 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER)) 2491 wrflags |= URWLOCK_WRITE_WAITERS; 2492 2493 for (;;) { 2494 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2495 /* try to lock it */ 2496 while (!(state & wrflags)) { 2497 if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) { 2498 umtx_key_release(&uq->uq_key); 2499 return (EAGAIN); 2500 } 2501 oldstate = casuword32(&rwlock->rw_state, state, state + 1); 2502 if (oldstate == state) { 2503 umtx_key_release(&uq->uq_key); 2504 return (0); 2505 } 2506 state = oldstate; 2507 } 2508 2509 if (error) 2510 break; 2511 2512 /* grab monitor lock */ 2513 umtxq_lock(&uq->uq_key); 2514 umtxq_busy(&uq->uq_key); 2515 umtxq_unlock(&uq->uq_key); 2516 2517 /* 2518 * re-read the state, in case it changed between the try-lock above 2519 * and the check below 2520 */ 2521 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2522 2523 /* set read contention bit */ 2524 while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) { 2525 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS); 2526 if (oldstate == state) 2527 goto sleep; 2528 state = oldstate; 2529 } 2530 2531 /* state is changed while setting flags, restart */ 2532 if (!(state & wrflags)) { 2533 umtxq_lock(&uq->uq_key); 2534 umtxq_unbusy(&uq->uq_key); 2535 umtxq_unlock(&uq->uq_key); 2536 continue; 2537 } 2538 2539 sleep: 2540 /* contention bit is set, before sleeping, increase read waiter count */ 2541 blocked_readers = fuword32(&rwlock->rw_blocked_readers); 2542 suword32(&rwlock->rw_blocked_readers, blocked_readers+1); 2543 2544 while (state & wrflags) { 2545 umtxq_lock(&uq->uq_key); 2546 umtxq_insert(uq); 2547 umtxq_unbusy(&uq->uq_key); 2548 2549 error = umtxq_sleep(uq, "urdlck", timo); 2550 2551 umtxq_busy(&uq->uq_key); 2552 umtxq_remove(uq); 2553 umtxq_unlock(&uq->uq_key); 2554 if (error) 2555 break; 2556 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2557 } 2558 2559 /* decrease read waiter count, and may clear read contention bit */ 2560 blocked_readers = fuword32(&rwlock->rw_blocked_readers); 2561 suword32(&rwlock->rw_blocked_readers, blocked_readers-1); 2562 if (blocked_readers == 1) { 2563 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2564 for (;;) { 2565 oldstate = casuword32(&rwlock->rw_state, state, 2566 state & ~URWLOCK_READ_WAITERS); 2567 if (oldstate == state) 2568 break; 2569 state = oldstate; 2570 } 2571 } 2572 2573 umtxq_lock(&uq->uq_key); 2574 umtxq_unbusy(&uq->uq_key); 2575 umtxq_unlock(&uq->uq_key); 2576 } 2577 umtx_key_release(&uq->uq_key); 2578 return (error); 2579 } 2580 2581 static int 2582 do_rw_rdlock2(struct thread *td, void *obj, long val, struct timespec *timeout) 2583 { 2584 struct timespec ts, ts2, ts3; 2585 struct timeval tv; 2586 int error; 2587 2588 getnanouptime(&ts); 2589 timespecadd(&ts, timeout); 2590 TIMESPEC_TO_TIMEVAL(&tv, timeout); 2591 for (;;) { 2592 error = do_rw_rdlock(td, obj, val, tvtohz(&tv)); 2593 if (error != ETIMEDOUT) 2594 break; 2595 getnanouptime(&ts2); 2596 if (timespeccmp(&ts2, &ts, >=)) { 2597 error = ETIMEDOUT; 2598 break; 2599 } 2600 ts3 = ts; 2601 timespecsub(&ts3, &ts2); 2602 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 2603 } 2604 if (error == ERESTART) 2605 error = EINTR; 2606 return (error); 2607 } 2608 2609 static int 2610 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, int timo) 2611 { 2612 struct umtx_q *uq; 2613 uint32_t flags; 2614 int32_t state, oldstate; 2615 int32_t blocked_writers; 2616 int32_t blocked_readers; 2617 int error; 2618 2619 uq = td->td_umtxq; 2620 flags = fuword32(&rwlock->rw_flags); 2621 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2622 if (error != 0) 2623 return (error); 2624 2625 blocked_readers = 0; 2626 for (;;) { 2627 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2628 while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { 2629 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER); 2630 if (oldstate == state) { 2631 umtx_key_release(&uq->uq_key); 2632 return (0); 2633 } 2634 state = oldstate; 2635 } 2636 2637 if (error) { 2638 if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) && 2639 blocked_readers != 0) { 2640 umtxq_lock(&uq->uq_key); 2641 umtxq_busy(&uq->uq_key); 2642 umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE); 2643 umtxq_unbusy(&uq->uq_key); 2644 umtxq_unlock(&uq->uq_key); 2645 } 2646 2647 break; 2648 } 2649 2650 /* grab monitor lock */ 2651 umtxq_lock(&uq->uq_key); 2652 umtxq_busy(&uq->uq_key); 2653 umtxq_unlock(&uq->uq_key); 2654 2655 /* 2656 * re-read the state, in case it changed between the try-lock above 2657 * and the check below 2658 */ 2659 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2660 2661 while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) && 2662 (state & URWLOCK_WRITE_WAITERS) == 0) { 2663 oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS); 2664 if (oldstate == state) 2665 goto sleep; 2666 state = oldstate; 2667 } 2668 2669 if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { 2670 umtxq_lock(&uq->uq_key); 2671 umtxq_unbusy(&uq->uq_key); 2672 umtxq_unlock(&uq->uq_key); 2673 continue; 2674 } 2675 sleep: 2676 blocked_writers = fuword32(&rwlock->rw_blocked_writers); 2677 suword32(&rwlock->rw_blocked_writers, blocked_writers+1); 2678 2679 while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) { 2680 umtxq_lock(&uq->uq_key); 2681 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE); 2682 umtxq_unbusy(&uq->uq_key); 2683 2684 error = umtxq_sleep(uq, "uwrlck", timo); 2685 2686 umtxq_busy(&uq->uq_key); 2687 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE); 2688 umtxq_unlock(&uq->uq_key); 2689 if (error) 2690 break; 2691 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2692 } 2693 2694 blocked_writers = fuword32(&rwlock->rw_blocked_writers); 2695 suword32(&rwlock->rw_blocked_writers, blocked_writers-1); 2696 if (blocked_writers == 1) { 2697 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2698 for (;;) { 2699 oldstate = casuword32(&rwlock->rw_state, state, 2700 state & ~URWLOCK_WRITE_WAITERS); 2701 if (oldstate == state) 2702 break; 2703 state = oldstate; 2704 } 2705 blocked_readers = fuword32(&rwlock->rw_blocked_readers); 2706 } else 2707 blocked_readers = 0; 2708 2709 umtxq_lock(&uq->uq_key); 2710 umtxq_unbusy(&uq->uq_key); 2711 umtxq_unlock(&uq->uq_key); 2712 } 2713 2714 umtx_key_release(&uq->uq_key); 2715 return (error); 2716 } 2717 2718 static int 2719 do_rw_wrlock2(struct thread *td, void *obj, struct timespec *timeout) 2720 { 2721 struct timespec ts, ts2, ts3; 2722 struct timeval tv; 2723 int error; 2724 2725 getnanouptime(&ts); 2726 timespecadd(&ts, timeout); 2727 TIMESPEC_TO_TIMEVAL(&tv, timeout); 2728 for (;;) { 2729 error = do_rw_wrlock(td, obj, tvtohz(&tv)); 2730 if (error != ETIMEDOUT) 2731 break; 2732 getnanouptime(&ts2); 2733 if (timespeccmp(&ts2, &ts, >=)) { 2734 error = ETIMEDOUT; 2735 break; 2736 } 2737 ts3 = ts; 2738 timespecsub(&ts3, &ts2); 2739 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 2740 } 2741 if (error == ERESTART) 2742 error = EINTR; 2743 return (error); 2744 } 2745 2746 static int 2747 do_rw_unlock(struct thread *td, struct urwlock *rwlock) 2748 { 2749 struct umtx_q *uq; 2750 uint32_t flags; 2751 int32_t state, oldstate; 2752 int error, q, count; 2753 2754 uq = td->td_umtxq; 2755 flags = fuword32(&rwlock->rw_flags); 2756 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2757 if (error != 0) 2758 return (error); 2759 2760 state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2761 if (state & URWLOCK_WRITE_OWNER) { 2762 for (;;) { 2763 oldstate = casuword32(&rwlock->rw_state, state, 2764 state & ~URWLOCK_WRITE_OWNER); 2765 if (oldstate != state) { 2766 state = oldstate; 2767 if (!(oldstate & URWLOCK_WRITE_OWNER)) { 2768 error = EPERM; 2769 goto out; 2770 } 2771 } else 2772 break; 2773 } 2774 } else if (URWLOCK_READER_COUNT(state) != 0) { 2775 for (;;) { 2776 oldstate = casuword32(&rwlock->rw_state, state, 2777 state - 1); 2778 if (oldstate != state) { 2779 state = oldstate; 2780 if (URWLOCK_READER_COUNT(oldstate) == 0) { 2781 error = EPERM; 2782 goto out; 2783 } 2784 } 2785 else 2786 break; 2787 } 2788 } else { 2789 error = EPERM; 2790 goto out; 2791 } 2792 2793 count = 0; 2794 2795 if (!(flags & URWLOCK_PREFER_READER)) { 2796 if (state & URWLOCK_WRITE_WAITERS) { 2797 count = 1; 2798 q = UMTX_EXCLUSIVE_QUEUE; 2799 } else if (state & URWLOCK_READ_WAITERS) { 2800 count = INT_MAX; 2801 q = UMTX_SHARED_QUEUE; 2802 } 2803 } else { 2804 if (state & URWLOCK_READ_WAITERS) { 2805 count = INT_MAX; 2806 q = UMTX_SHARED_QUEUE; 2807 } else if (state & URWLOCK_WRITE_WAITERS) { 2808 count = 1; 2809 q = UMTX_EXCLUSIVE_QUEUE; 2810 } 2811 } 2812 2813 if (count) { 2814 umtxq_lock(&uq->uq_key); 2815 umtxq_busy(&uq->uq_key); 2816 umtxq_signal_queue(&uq->uq_key, count, q); 2817 umtxq_unbusy(&uq->uq_key); 2818 umtxq_unlock(&uq->uq_key); 2819 } 2820 out: 2821 umtx_key_release(&uq->uq_key); 2822 return (error); 2823 } 2824 2825 static int 2826 do_sem_wait(struct thread *td, struct _usem *sem, struct timespec *timeout) 2827 { 2828 struct umtx_q *uq; 2829 struct timeval tv; 2830 struct timespec cts, ets, tts; 2831 uint32_t flags, count; 2832 int error; 2833 2834 uq = td->td_umtxq; 2835 flags = fuword32(&sem->_flags); 2836 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); 2837 if (error != 0) 2838 return (error); 2839 umtxq_lock(&uq->uq_key); 2840 umtxq_busy(&uq->uq_key); 2841 umtxq_insert(uq); 2842 umtxq_unlock(&uq->uq_key); 2843 2844 if (fuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters)) == 0) 2845 casuword32(__DEVOLATILE(uint32_t *, &sem->_has_waiters), 0, 1); 2846 2847 count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count)); 2848 if (count != 0) { 2849 umtxq_lock(&uq->uq_key); 2850 umtxq_unbusy(&uq->uq_key); 2851 umtxq_remove(uq); 2852 umtxq_unlock(&uq->uq_key); 2853 umtx_key_release(&uq->uq_key); 2854 return (0); 2855 } 2856 2857 umtxq_lock(&uq->uq_key); 2858 umtxq_unbusy(&uq->uq_key); 2859 umtxq_unlock(&uq->uq_key); 2860 2861 umtxq_lock(&uq->uq_key); 2862 if (timeout == NULL) { 2863 error = umtxq_sleep(uq, "usem", 0); 2864 } else { 2865 getnanouptime(&ets); 2866 timespecadd(&ets, timeout); 2867 TIMESPEC_TO_TIMEVAL(&tv, timeout); 2868 for (;;) { 2869 error = umtxq_sleep(uq, "usem", tvtohz(&tv)); 2870 if (error != ETIMEDOUT) 2871 break; 2872 getnanouptime(&cts); 2873 if (timespeccmp(&cts, &ets, >=)) { 2874 error = ETIMEDOUT; 2875 break; 2876 } 2877 tts = ets; 2878 timespecsub(&tts, &cts); 2879 TIMESPEC_TO_TIMEVAL(&tv, &tts); 2880 } 2881 } 2882 2883 if ((uq->uq_flags & UQF_UMTXQ) == 0) 2884 error = 0; 2885 else { 2886 umtxq_remove(uq); 2887 if (error == ERESTART) 2888 error = EINTR; 2889 } 2890 umtxq_unlock(&uq->uq_key); 2891 umtx_key_release(&uq->uq_key); 2892 return (error); 2893 } 2894 2895 /* 2896 * Signal a userland condition variable. 2897 */ 2898 static int 2899 do_sem_wake(struct thread *td, struct _usem *sem) 2900 { 2901 struct umtx_key key; 2902 int error, cnt, nwake; 2903 uint32_t flags; 2904 2905 flags = fuword32(&sem->_flags); 2906 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) 2907 return (error); 2908 umtxq_lock(&key); 2909 umtxq_busy(&key); 2910 cnt = umtxq_count(&key); 2911 nwake = umtxq_signal(&key, 1); 2912 if (cnt <= nwake) { 2913 umtxq_unlock(&key); 2914 error = suword32( 2915 __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0); 2916 umtxq_lock(&key); 2917 } 2918 umtxq_unbusy(&key); 2919 umtxq_unlock(&key); 2920 umtx_key_release(&key); 2921 return (error); 2922 } 2923 2924 int 2925 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 2926 /* struct umtx *umtx */ 2927 { 2928 return _do_lock_umtx(td, uap->umtx, td->td_tid, 0); 2929 } 2930 2931 int 2932 _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 2933 /* struct umtx *umtx */ 2934 { 2935 return do_unlock_umtx(td, uap->umtx, td->td_tid); 2936 } 2937 2938 static int 2939 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap) 2940 { 2941 struct timespec *ts, timeout; 2942 int error; 2943 2944 /* Allow a null timespec (wait forever). */ 2945 if (uap->uaddr2 == NULL) 2946 ts = NULL; 2947 else { 2948 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2949 if (error != 0) 2950 return (error); 2951 if (timeout.tv_nsec >= 1000000000 || 2952 timeout.tv_nsec < 0) { 2953 return (EINVAL); 2954 } 2955 ts = &timeout; 2956 } 2957 return (do_lock_umtx(td, uap->obj, uap->val, ts)); 2958 } 2959 2960 static int 2961 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap) 2962 { 2963 return (do_unlock_umtx(td, uap->obj, uap->val)); 2964 } 2965 2966 static int 2967 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) 2968 { 2969 struct timespec *ts, timeout; 2970 int error; 2971 2972 if (uap->uaddr2 == NULL) 2973 ts = NULL; 2974 else { 2975 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2976 if (error != 0) 2977 return (error); 2978 if (timeout.tv_nsec >= 1000000000 || 2979 timeout.tv_nsec < 0) 2980 return (EINVAL); 2981 ts = &timeout; 2982 } 2983 return do_wait(td, uap->obj, uap->val, ts, 0, 0); 2984 } 2985 2986 static int 2987 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap) 2988 { 2989 struct timespec *ts, timeout; 2990 int error; 2991 2992 if (uap->uaddr2 == NULL) 2993 ts = NULL; 2994 else { 2995 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2996 if (error != 0) 2997 return (error); 2998 if (timeout.tv_nsec >= 1000000000 || 2999 timeout.tv_nsec < 0) 3000 return (EINVAL); 3001 ts = &timeout; 3002 } 3003 return do_wait(td, uap->obj, uap->val, ts, 1, 0); 3004 } 3005 3006 static int 3007 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) 3008 { 3009 struct timespec *ts, timeout; 3010 int error; 3011 3012 if (uap->uaddr2 == NULL) 3013 ts = NULL; 3014 else { 3015 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 3016 if (error != 0) 3017 return (error); 3018 if (timeout.tv_nsec >= 1000000000 || 3019 timeout.tv_nsec < 0) 3020 return (EINVAL); 3021 ts = &timeout; 3022 } 3023 return do_wait(td, uap->obj, uap->val, ts, 1, 1); 3024 } 3025 3026 static int 3027 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) 3028 { 3029 return (kern_umtx_wake(td, uap->obj, uap->val, 0)); 3030 } 3031 3032 static int 3033 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) 3034 { 3035 return (kern_umtx_wake(td, uap->obj, uap->val, 1)); 3036 } 3037 3038 static int 3039 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) 3040 { 3041 struct timespec *ts, timeout; 3042 int error; 3043 3044 /* Allow a null timespec (wait forever). */ 3045 if (uap->uaddr2 == NULL) 3046 ts = NULL; 3047 else { 3048 error = copyin(uap->uaddr2, &timeout, 3049 sizeof(timeout)); 3050 if (error != 0) 3051 return (error); 3052 if (timeout.tv_nsec >= 1000000000 || 3053 timeout.tv_nsec < 0) { 3054 return (EINVAL); 3055 } 3056 ts = &timeout; 3057 } 3058 return do_lock_umutex(td, uap->obj, ts, 0); 3059 } 3060 3061 static int 3062 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) 3063 { 3064 return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY); 3065 } 3066 3067 static int 3068 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) 3069 { 3070 struct timespec *ts, timeout; 3071 int error; 3072 3073 /* Allow a null timespec (wait forever). */ 3074 if (uap->uaddr2 == NULL) 3075 ts = NULL; 3076 else { 3077 error = copyin(uap->uaddr2, &timeout, 3078 sizeof(timeout)); 3079 if (error != 0) 3080 return (error); 3081 if (timeout.tv_nsec >= 1000000000 || 3082 timeout.tv_nsec < 0) { 3083 return (EINVAL); 3084 } 3085 ts = &timeout; 3086 } 3087 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT); 3088 } 3089 3090 static int 3091 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) 3092 { 3093 return do_wake_umutex(td, uap->obj); 3094 } 3095 3096 static int 3097 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) 3098 { 3099 return do_unlock_umutex(td, uap->obj); 3100 } 3101 3102 static int 3103 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) 3104 { 3105 return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1); 3106 } 3107 3108 static int 3109 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) 3110 { 3111 struct timespec *ts, timeout; 3112 int error; 3113 3114 /* Allow a null timespec (wait forever). */ 3115 if (uap->uaddr2 == NULL) 3116 ts = NULL; 3117 else { 3118 error = copyin(uap->uaddr2, &timeout, 3119 sizeof(timeout)); 3120 if (error != 0) 3121 return (error); 3122 if (timeout.tv_nsec >= 1000000000 || 3123 timeout.tv_nsec < 0) { 3124 return (EINVAL); 3125 } 3126 ts = &timeout; 3127 } 3128 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 3129 } 3130 3131 static int 3132 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) 3133 { 3134 return do_cv_signal(td, uap->obj); 3135 } 3136 3137 static int 3138 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) 3139 { 3140 return do_cv_broadcast(td, uap->obj); 3141 } 3142 3143 static int 3144 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap) 3145 { 3146 struct timespec timeout; 3147 int error; 3148 3149 /* Allow a null timespec (wait forever). */ 3150 if (uap->uaddr2 == NULL) { 3151 error = do_rw_rdlock(td, uap->obj, uap->val, 0); 3152 } else { 3153 error = copyin(uap->uaddr2, &timeout, 3154 sizeof(timeout)); 3155 if (error != 0) 3156 return (error); 3157 if (timeout.tv_nsec >= 1000000000 || 3158 timeout.tv_nsec < 0) { 3159 return (EINVAL); 3160 } 3161 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout); 3162 } 3163 return (error); 3164 } 3165 3166 static int 3167 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap) 3168 { 3169 struct timespec timeout; 3170 int error; 3171 3172 /* Allow a null timespec (wait forever). */ 3173 if (uap->uaddr2 == NULL) { 3174 error = do_rw_wrlock(td, uap->obj, 0); 3175 } else { 3176 error = copyin(uap->uaddr2, &timeout, 3177 sizeof(timeout)); 3178 if (error != 0) 3179 return (error); 3180 if (timeout.tv_nsec >= 1000000000 || 3181 timeout.tv_nsec < 0) { 3182 return (EINVAL); 3183 } 3184 3185 error = do_rw_wrlock2(td, uap->obj, &timeout); 3186 } 3187 return (error); 3188 } 3189 3190 static int 3191 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) 3192 { 3193 return do_rw_unlock(td, uap->obj); 3194 } 3195 3196 static int 3197 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap) 3198 { 3199 struct timespec *ts, timeout; 3200 int error; 3201 3202 /* Allow a null timespec (wait forever). */ 3203 if (uap->uaddr2 == NULL) 3204 ts = NULL; 3205 else { 3206 error = copyin(uap->uaddr2, &timeout, 3207 sizeof(timeout)); 3208 if (error != 0) 3209 return (error); 3210 if (timeout.tv_nsec >= 1000000000 || 3211 timeout.tv_nsec < 0) { 3212 return (EINVAL); 3213 } 3214 ts = &timeout; 3215 } 3216 return (do_sem_wait(td, uap->obj, ts)); 3217 } 3218 3219 static int 3220 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap) 3221 { 3222 return do_sem_wake(td, uap->obj); 3223 } 3224 3225 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); 3226 3227 static _umtx_op_func op_table[] = { 3228 __umtx_op_lock_umtx, /* UMTX_OP_LOCK */ 3229 __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */ 3230 __umtx_op_wait, /* UMTX_OP_WAIT */ 3231 __umtx_op_wake, /* UMTX_OP_WAKE */ 3232 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */ 3233 __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */ 3234 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ 3235 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ 3236 __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/ 3237 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ 3238 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */ 3239 __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */ 3240 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */ 3241 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */ 3242 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ 3243 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */ 3244 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ 3245 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */ 3246 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */ 3247 __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */ 3248 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */ 3249 }; 3250 3251 int 3252 _umtx_op(struct thread *td, struct _umtx_op_args *uap) 3253 { 3254 if ((unsigned)uap->op < UMTX_OP_MAX) 3255 return (*op_table[uap->op])(td, uap); 3256 return (EINVAL); 3257 } 3258 3259 #ifdef COMPAT_FREEBSD32 3260 int 3261 freebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap) 3262 /* struct umtx *umtx */ 3263 { 3264 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL)); 3265 } 3266 3267 int 3268 freebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap) 3269 /* struct umtx *umtx */ 3270 { 3271 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid)); 3272 } 3273 3274 struct timespec32 { 3275 uint32_t tv_sec; 3276 uint32_t tv_nsec; 3277 }; 3278 3279 static inline int 3280 copyin_timeout32(void *addr, struct timespec *tsp) 3281 { 3282 struct timespec32 ts32; 3283 int error; 3284 3285 error = copyin(addr, &ts32, sizeof(struct timespec32)); 3286 if (error == 0) { 3287 tsp->tv_sec = ts32.tv_sec; 3288 tsp->tv_nsec = ts32.tv_nsec; 3289 } 3290 return (error); 3291 } 3292 3293 static int 3294 __umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) 3295 { 3296 struct timespec *ts, timeout; 3297 int error; 3298 3299 /* Allow a null timespec (wait forever). */ 3300 if (uap->uaddr2 == NULL) 3301 ts = NULL; 3302 else { 3303 error = copyin_timeout32(uap->uaddr2, &timeout); 3304 if (error != 0) 3305 return (error); 3306 if (timeout.tv_nsec >= 1000000000 || 3307 timeout.tv_nsec < 0) { 3308 return (EINVAL); 3309 } 3310 ts = &timeout; 3311 } 3312 return (do_lock_umtx32(td, uap->obj, uap->val, ts)); 3313 } 3314 3315 static int 3316 __umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) 3317 { 3318 return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val)); 3319 } 3320 3321 static int 3322 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 3323 { 3324 struct timespec *ts, timeout; 3325 int error; 3326 3327 if (uap->uaddr2 == NULL) 3328 ts = NULL; 3329 else { 3330 error = copyin_timeout32(uap->uaddr2, &timeout); 3331 if (error != 0) 3332 return (error); 3333 if (timeout.tv_nsec >= 1000000000 || 3334 timeout.tv_nsec < 0) 3335 return (EINVAL); 3336 ts = &timeout; 3337 } 3338 return do_wait(td, uap->obj, uap->val, ts, 1, 0); 3339 } 3340 3341 static int 3342 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 3343 { 3344 struct timespec *ts, timeout; 3345 int error; 3346 3347 /* Allow a null timespec (wait forever). */ 3348 if (uap->uaddr2 == NULL) 3349 ts = NULL; 3350 else { 3351 error = copyin_timeout32(uap->uaddr2, &timeout); 3352 if (error != 0) 3353 return (error); 3354 if (timeout.tv_nsec >= 1000000000 || 3355 timeout.tv_nsec < 0) 3356 return (EINVAL); 3357 ts = &timeout; 3358 } 3359 return do_lock_umutex(td, uap->obj, ts, 0); 3360 } 3361 3362 static int 3363 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 3364 { 3365 struct timespec *ts, timeout; 3366 int error; 3367 3368 /* Allow a null timespec (wait forever). */ 3369 if (uap->uaddr2 == NULL) 3370 ts = NULL; 3371 else { 3372 error = copyin_timeout32(uap->uaddr2, &timeout); 3373 if (error != 0) 3374 return (error); 3375 if (timeout.tv_nsec >= 1000000000 || 3376 timeout.tv_nsec < 0) 3377 return (EINVAL); 3378 ts = &timeout; 3379 } 3380 return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT); 3381 } 3382 3383 static int 3384 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 3385 { 3386 struct timespec *ts, timeout; 3387 int error; 3388 3389 /* Allow a null timespec (wait forever). */ 3390 if (uap->uaddr2 == NULL) 3391 ts = NULL; 3392 else { 3393 error = copyin_timeout32(uap->uaddr2, &timeout); 3394 if (error != 0) 3395 return (error); 3396 if (timeout.tv_nsec >= 1000000000 || 3397 timeout.tv_nsec < 0) 3398 return (EINVAL); 3399 ts = &timeout; 3400 } 3401 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 3402 } 3403 3404 static int 3405 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap) 3406 { 3407 struct timespec timeout; 3408 int error; 3409 3410 /* Allow a null timespec (wait forever). */ 3411 if (uap->uaddr2 == NULL) { 3412 error = do_rw_rdlock(td, uap->obj, uap->val, 0); 3413 } else { 3414 error = copyin_timeout32(uap->uaddr2, &timeout); 3415 if (error != 0) 3416 return (error); 3417 if (timeout.tv_nsec >= 1000000000 || 3418 timeout.tv_nsec < 0) { 3419 return (EINVAL); 3420 } 3421 error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout); 3422 } 3423 return (error); 3424 } 3425 3426 static int 3427 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap) 3428 { 3429 struct timespec timeout; 3430 int error; 3431 3432 /* Allow a null timespec (wait forever). */ 3433 if (uap->uaddr2 == NULL) { 3434 error = do_rw_wrlock(td, uap->obj, 0); 3435 } else { 3436 error = copyin_timeout32(uap->uaddr2, &timeout); 3437 if (error != 0) 3438 return (error); 3439 if (timeout.tv_nsec >= 1000000000 || 3440 timeout.tv_nsec < 0) { 3441 return (EINVAL); 3442 } 3443 3444 error = do_rw_wrlock2(td, uap->obj, &timeout); 3445 } 3446 return (error); 3447 } 3448 3449 static int 3450 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap) 3451 { 3452 struct timespec *ts, timeout; 3453 int error; 3454 3455 if (uap->uaddr2 == NULL) 3456 ts = NULL; 3457 else { 3458 error = copyin_timeout32(uap->uaddr2, &timeout); 3459 if (error != 0) 3460 return (error); 3461 if (timeout.tv_nsec >= 1000000000 || 3462 timeout.tv_nsec < 0) 3463 return (EINVAL); 3464 ts = &timeout; 3465 } 3466 return do_wait(td, uap->obj, uap->val, ts, 1, 1); 3467 } 3468 3469 static int 3470 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 3471 { 3472 struct timespec *ts, timeout; 3473 int error; 3474 3475 /* Allow a null timespec (wait forever). */ 3476 if (uap->uaddr2 == NULL) 3477 ts = NULL; 3478 else { 3479 error = copyin_timeout32(uap->uaddr2, &timeout); 3480 if (error != 0) 3481 return (error); 3482 if (timeout.tv_nsec >= 1000000000 || 3483 timeout.tv_nsec < 0) 3484 return (EINVAL); 3485 ts = &timeout; 3486 } 3487 return (do_sem_wait(td, uap->obj, ts)); 3488 } 3489 3490 static _umtx_op_func op_table_compat32[] = { 3491 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */ 3492 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */ 3493 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */ 3494 __umtx_op_wake, /* UMTX_OP_WAKE */ 3495 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */ 3496 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */ 3497 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ 3498 __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ 3499 __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/ 3500 __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ 3501 __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */ 3502 __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */ 3503 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */ 3504 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */ 3505 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ 3506 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */ 3507 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ 3508 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */ 3509 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */ 3510 __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */ 3511 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */ 3512 }; 3513 3514 int 3515 freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap) 3516 { 3517 if ((unsigned)uap->op < UMTX_OP_MAX) 3518 return (*op_table_compat32[uap->op])(td, 3519 (struct _umtx_op_args *)uap); 3520 return (EINVAL); 3521 } 3522 #endif 3523 3524 void 3525 umtx_thread_init(struct thread *td) 3526 { 3527 td->td_umtxq = umtxq_alloc(); 3528 td->td_umtxq->uq_thread = td; 3529 } 3530 3531 void 3532 umtx_thread_fini(struct thread *td) 3533 { 3534 umtxq_free(td->td_umtxq); 3535 } 3536 3537 /* 3538 * It will be called when new thread is created, e.g fork(). 3539 */ 3540 void 3541 umtx_thread_alloc(struct thread *td) 3542 { 3543 struct umtx_q *uq; 3544 3545 uq = td->td_umtxq; 3546 uq->uq_inherited_pri = PRI_MAX; 3547 3548 KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); 3549 KASSERT(uq->uq_thread == td, ("uq_thread != td")); 3550 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); 3551 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); 3552 } 3553 3554 /* 3555 * exec() hook. 3556 */ 3557 static void 3558 umtx_exec_hook(void *arg __unused, struct proc *p __unused, 3559 struct image_params *imgp __unused) 3560 { 3561 umtx_thread_cleanup(curthread); 3562 } 3563 3564 /* 3565 * thread_exit() hook. 3566 */ 3567 void 3568 umtx_thread_exit(struct thread *td) 3569 { 3570 umtx_thread_cleanup(td); 3571 } 3572 3573 /* 3574 * clean up umtx data. 3575 */ 3576 static void 3577 umtx_thread_cleanup(struct thread *td) 3578 { 3579 struct umtx_q *uq; 3580 struct umtx_pi *pi; 3581 3582 if ((uq = td->td_umtxq) == NULL) 3583 return; 3584 3585 mtx_lock_spin(&umtx_lock); 3586 uq->uq_inherited_pri = PRI_MAX; 3587 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { 3588 pi->pi_owner = NULL; 3589 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); 3590 } 3591 mtx_unlock_spin(&umtx_lock); 3592 thread_lock(td); 3593 sched_unlend_user_prio(td, PRI_MAX); 3594 thread_unlock(td); 3595 } 3596