1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2015, 2016 The FreeBSD Foundation 5 * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 6 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 7 * All rights reserved. 8 * 9 * Portions of this software were developed by Konstantin Belousov 10 * under sponsorship from the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice unmodified, this list of conditions, and the following 17 * disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_umtx_profiling.h" 38 39 #include <sys/param.h> 40 #include <sys/kernel.h> 41 #include <sys/fcntl.h> 42 #include <sys/file.h> 43 #include <sys/filedesc.h> 44 #include <sys/limits.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/mutex.h> 49 #include <sys/priv.h> 50 #include <sys/proc.h> 51 #include <sys/resource.h> 52 #include <sys/resourcevar.h> 53 #include <sys/rwlock.h> 54 #include <sys/sbuf.h> 55 #include <sys/sched.h> 56 #include <sys/smp.h> 57 #include <sys/sysctl.h> 58 #include <sys/sysent.h> 59 #include <sys/systm.h> 60 #include <sys/sysproto.h> 61 #include <sys/syscallsubr.h> 62 #include <sys/taskqueue.h> 63 #include <sys/time.h> 64 #include <sys/eventhandler.h> 65 #include <sys/umtx.h> 66 67 #include <security/mac/mac_framework.h> 68 69 #include <vm/vm.h> 70 #include <vm/vm_param.h> 71 #include <vm/pmap.h> 72 #include <vm/vm_map.h> 73 #include <vm/vm_object.h> 74 75 #include <machine/atomic.h> 76 #include <machine/cpu.h> 77 78 #ifdef COMPAT_FREEBSD32 79 #include <compat/freebsd32/freebsd32_proto.h> 80 #endif 81 82 #define _UMUTEX_TRY 1 83 #define _UMUTEX_WAIT 2 84 85 #ifdef UMTX_PROFILING 86 #define UPROF_PERC_BIGGER(w, f, sw, sf) \ 87 (((w) > (sw)) || ((w) == (sw) && (f) > (sf))) 88 #endif 89 90 /* Priority inheritance mutex info. */ 91 struct umtx_pi { 92 /* Owner thread */ 93 struct thread *pi_owner; 94 95 /* Reference count */ 96 int pi_refcount; 97 98 /* List entry to link umtx holding by thread */ 99 TAILQ_ENTRY(umtx_pi) pi_link; 100 101 /* List entry in hash */ 102 TAILQ_ENTRY(umtx_pi) pi_hashlink; 103 104 /* List for waiters */ 105 TAILQ_HEAD(,umtx_q) pi_blocked; 106 107 /* Identify a userland lock object */ 108 struct umtx_key pi_key; 109 }; 110 111 /* A userland synchronous object user. */ 112 struct umtx_q { 113 /* Linked list for the hash. */ 114 TAILQ_ENTRY(umtx_q) uq_link; 115 116 /* Umtx key. */ 117 struct umtx_key uq_key; 118 119 /* Umtx flags. */ 120 int uq_flags; 121 #define UQF_UMTXQ 0x0001 122 123 /* The thread waits on. */ 124 struct thread *uq_thread; 125 126 /* 127 * Blocked on PI mutex. read can use chain lock 128 * or umtx_lock, write must have both chain lock and 129 * umtx_lock being hold. 130 */ 131 struct umtx_pi *uq_pi_blocked; 132 133 /* On blocked list */ 134 TAILQ_ENTRY(umtx_q) uq_lockq; 135 136 /* Thread contending with us */ 137 TAILQ_HEAD(,umtx_pi) uq_pi_contested; 138 139 /* Inherited priority from PP mutex */ 140 u_char uq_inherited_pri; 141 142 /* Spare queue ready to be reused */ 143 struct umtxq_queue *uq_spare_queue; 144 145 /* The queue we on */ 146 struct umtxq_queue *uq_cur_queue; 147 }; 148 149 TAILQ_HEAD(umtxq_head, umtx_q); 150 151 /* Per-key wait-queue */ 152 struct umtxq_queue { 153 struct umtxq_head head; 154 struct umtx_key key; 155 LIST_ENTRY(umtxq_queue) link; 156 int length; 157 }; 158 159 LIST_HEAD(umtxq_list, umtxq_queue); 160 161 /* Userland lock object's wait-queue chain */ 162 struct umtxq_chain { 163 /* Lock for this chain. */ 164 struct mtx uc_lock; 165 166 /* List of sleep queues. */ 167 struct umtxq_list uc_queue[2]; 168 #define UMTX_SHARED_QUEUE 0 169 #define UMTX_EXCLUSIVE_QUEUE 1 170 171 LIST_HEAD(, umtxq_queue) uc_spare_queue; 172 173 /* Busy flag */ 174 char uc_busy; 175 176 /* Chain lock waiters */ 177 int uc_waiters; 178 179 /* All PI in the list */ 180 TAILQ_HEAD(,umtx_pi) uc_pi_list; 181 182 #ifdef UMTX_PROFILING 183 u_int length; 184 u_int max_length; 185 #endif 186 }; 187 188 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) 189 190 /* 191 * Don't propagate time-sharing priority, there is a security reason, 192 * a user can simply introduce PI-mutex, let thread A lock the mutex, 193 * and let another thread B block on the mutex, because B is 194 * sleeping, its priority will be boosted, this causes A's priority to 195 * be boosted via priority propagating too and will never be lowered even 196 * if it is using 100%CPU, this is unfair to other processes. 197 */ 198 199 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ 200 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ 201 PRI_MAX_TIMESHARE : (td)->td_user_pri) 202 203 #define GOLDEN_RATIO_PRIME 2654404609U 204 #ifndef UMTX_CHAINS 205 #define UMTX_CHAINS 512 206 #endif 207 #define UMTX_SHIFTS (__WORD_BIT - 9) 208 209 #define GET_SHARE(flags) \ 210 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) 211 212 #define BUSY_SPINS 200 213 214 struct abs_timeout { 215 int clockid; 216 bool is_abs_real; /* TIMER_ABSTIME && CLOCK_REALTIME* */ 217 struct timespec cur; 218 struct timespec end; 219 }; 220 221 #ifdef COMPAT_FREEBSD32 222 struct umutex32 { 223 volatile __lwpid_t m_owner; /* Owner of the mutex */ 224 __uint32_t m_flags; /* Flags of the mutex */ 225 __uint32_t m_ceilings[2]; /* Priority protect ceiling */ 226 __uint32_t m_rb_lnk; /* Robust linkage */ 227 __uint32_t m_pad; 228 __uint32_t m_spare[2]; 229 }; 230 231 _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32"); 232 _Static_assert(__offsetof(struct umutex, m_spare[0]) == 233 __offsetof(struct umutex32, m_spare[0]), "m_spare32"); 234 #endif 235 236 int umtx_shm_vnobj_persistent = 0; 237 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN, 238 &umtx_shm_vnobj_persistent, 0, 239 "False forces destruction of umtx attached to file, on last close"); 240 static int umtx_max_rb = 1000; 241 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN, 242 &umtx_max_rb, 0, 243 ""); 244 245 static uma_zone_t umtx_pi_zone; 246 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; 247 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); 248 static int umtx_pi_allocated; 249 250 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); 251 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, 252 &umtx_pi_allocated, 0, "Allocated umtx_pi"); 253 static int umtx_verbose_rb = 1; 254 SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN, 255 &umtx_verbose_rb, 0, 256 ""); 257 258 #ifdef UMTX_PROFILING 259 static long max_length; 260 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length"); 261 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD, 0, "umtx chain stats"); 262 #endif 263 264 static void abs_timeout_update(struct abs_timeout *timo); 265 266 static void umtx_shm_init(void); 267 static void umtxq_sysinit(void *); 268 static void umtxq_hash(struct umtx_key *key); 269 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); 270 static void umtxq_lock(struct umtx_key *key); 271 static void umtxq_unlock(struct umtx_key *key); 272 static void umtxq_busy(struct umtx_key *key); 273 static void umtxq_unbusy(struct umtx_key *key); 274 static void umtxq_insert_queue(struct umtx_q *uq, int q); 275 static void umtxq_remove_queue(struct umtx_q *uq, int q); 276 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *); 277 static int umtxq_count(struct umtx_key *key); 278 static struct umtx_pi *umtx_pi_alloc(int); 279 static void umtx_pi_free(struct umtx_pi *pi); 280 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, 281 bool rb); 282 static void umtx_thread_cleanup(struct thread *td); 283 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, 284 struct image_params *imgp __unused); 285 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); 286 287 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE) 288 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE) 289 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE) 290 291 static struct mtx umtx_lock; 292 293 #ifdef UMTX_PROFILING 294 static void 295 umtx_init_profiling(void) 296 { 297 struct sysctl_oid *chain_oid; 298 char chain_name[10]; 299 int i; 300 301 for (i = 0; i < UMTX_CHAINS; ++i) { 302 snprintf(chain_name, sizeof(chain_name), "%d", i); 303 chain_oid = SYSCTL_ADD_NODE(NULL, 304 SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO, 305 chain_name, CTLFLAG_RD, NULL, "umtx hash stats"); 306 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO, 307 "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL); 308 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO, 309 "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL); 310 } 311 } 312 313 static int 314 sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS) 315 { 316 char buf[512]; 317 struct sbuf sb; 318 struct umtxq_chain *uc; 319 u_int fract, i, j, tot, whole; 320 u_int sf0, sf1, sf2, sf3, sf4; 321 u_int si0, si1, si2, si3, si4; 322 u_int sw0, sw1, sw2, sw3, sw4; 323 324 sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN); 325 for (i = 0; i < 2; i++) { 326 tot = 0; 327 for (j = 0; j < UMTX_CHAINS; ++j) { 328 uc = &umtxq_chains[i][j]; 329 mtx_lock(&uc->uc_lock); 330 tot += uc->max_length; 331 mtx_unlock(&uc->uc_lock); 332 } 333 if (tot == 0) 334 sbuf_printf(&sb, "%u) Empty ", i); 335 else { 336 sf0 = sf1 = sf2 = sf3 = sf4 = 0; 337 si0 = si1 = si2 = si3 = si4 = 0; 338 sw0 = sw1 = sw2 = sw3 = sw4 = 0; 339 for (j = 0; j < UMTX_CHAINS; j++) { 340 uc = &umtxq_chains[i][j]; 341 mtx_lock(&uc->uc_lock); 342 whole = uc->max_length * 100; 343 mtx_unlock(&uc->uc_lock); 344 fract = (whole % tot) * 100; 345 if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) { 346 sf0 = fract; 347 si0 = j; 348 sw0 = whole; 349 } else if (UPROF_PERC_BIGGER(whole, fract, sw1, 350 sf1)) { 351 sf1 = fract; 352 si1 = j; 353 sw1 = whole; 354 } else if (UPROF_PERC_BIGGER(whole, fract, sw2, 355 sf2)) { 356 sf2 = fract; 357 si2 = j; 358 sw2 = whole; 359 } else if (UPROF_PERC_BIGGER(whole, fract, sw3, 360 sf3)) { 361 sf3 = fract; 362 si3 = j; 363 sw3 = whole; 364 } else if (UPROF_PERC_BIGGER(whole, fract, sw4, 365 sf4)) { 366 sf4 = fract; 367 si4 = j; 368 sw4 = whole; 369 } 370 } 371 sbuf_printf(&sb, "queue %u:\n", i); 372 sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot, 373 sf0 / tot, si0); 374 sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot, 375 sf1 / tot, si1); 376 sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot, 377 sf2 / tot, si2); 378 sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot, 379 sf3 / tot, si3); 380 sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot, 381 sf4 / tot, si4); 382 } 383 } 384 sbuf_trim(&sb); 385 sbuf_finish(&sb); 386 sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 387 sbuf_delete(&sb); 388 return (0); 389 } 390 391 static int 392 sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS) 393 { 394 struct umtxq_chain *uc; 395 u_int i, j; 396 int clear, error; 397 398 clear = 0; 399 error = sysctl_handle_int(oidp, &clear, 0, req); 400 if (error != 0 || req->newptr == NULL) 401 return (error); 402 403 if (clear != 0) { 404 for (i = 0; i < 2; ++i) { 405 for (j = 0; j < UMTX_CHAINS; ++j) { 406 uc = &umtxq_chains[i][j]; 407 mtx_lock(&uc->uc_lock); 408 uc->length = 0; 409 uc->max_length = 0; 410 mtx_unlock(&uc->uc_lock); 411 } 412 } 413 } 414 return (0); 415 } 416 417 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear, 418 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, 419 sysctl_debug_umtx_chains_clear, "I", "Clear umtx chains statistics"); 420 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks, 421 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0, 422 sysctl_debug_umtx_chains_peaks, "A", "Highest peaks in chains max length"); 423 #endif 424 425 static void 426 umtxq_sysinit(void *arg __unused) 427 { 428 int i, j; 429 430 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), 431 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 432 for (i = 0; i < 2; ++i) { 433 for (j = 0; j < UMTX_CHAINS; ++j) { 434 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL, 435 MTX_DEF | MTX_DUPOK); 436 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]); 437 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]); 438 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue); 439 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list); 440 umtxq_chains[i][j].uc_busy = 0; 441 umtxq_chains[i][j].uc_waiters = 0; 442 #ifdef UMTX_PROFILING 443 umtxq_chains[i][j].length = 0; 444 umtxq_chains[i][j].max_length = 0; 445 #endif 446 } 447 } 448 #ifdef UMTX_PROFILING 449 umtx_init_profiling(); 450 #endif 451 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_DEF); 452 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, 453 EVENTHANDLER_PRI_ANY); 454 umtx_shm_init(); 455 } 456 457 struct umtx_q * 458 umtxq_alloc(void) 459 { 460 struct umtx_q *uq; 461 462 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); 463 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, 464 M_WAITOK | M_ZERO); 465 TAILQ_INIT(&uq->uq_spare_queue->head); 466 TAILQ_INIT(&uq->uq_pi_contested); 467 uq->uq_inherited_pri = PRI_MAX; 468 return (uq); 469 } 470 471 void 472 umtxq_free(struct umtx_q *uq) 473 { 474 475 MPASS(uq->uq_spare_queue != NULL); 476 free(uq->uq_spare_queue, M_UMTX); 477 free(uq, M_UMTX); 478 } 479 480 static inline void 481 umtxq_hash(struct umtx_key *key) 482 { 483 unsigned n; 484 485 n = (uintptr_t)key->info.both.a + key->info.both.b; 486 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; 487 } 488 489 static inline struct umtxq_chain * 490 umtxq_getchain(struct umtx_key *key) 491 { 492 493 if (key->type <= TYPE_SEM) 494 return (&umtxq_chains[1][key->hash]); 495 return (&umtxq_chains[0][key->hash]); 496 } 497 498 /* 499 * Lock a chain. 500 */ 501 static inline void 502 umtxq_lock(struct umtx_key *key) 503 { 504 struct umtxq_chain *uc; 505 506 uc = umtxq_getchain(key); 507 mtx_lock(&uc->uc_lock); 508 } 509 510 /* 511 * Unlock a chain. 512 */ 513 static inline void 514 umtxq_unlock(struct umtx_key *key) 515 { 516 struct umtxq_chain *uc; 517 518 uc = umtxq_getchain(key); 519 mtx_unlock(&uc->uc_lock); 520 } 521 522 /* 523 * Set chain to busy state when following operation 524 * may be blocked (kernel mutex can not be used). 525 */ 526 static inline void 527 umtxq_busy(struct umtx_key *key) 528 { 529 struct umtxq_chain *uc; 530 531 uc = umtxq_getchain(key); 532 mtx_assert(&uc->uc_lock, MA_OWNED); 533 if (uc->uc_busy) { 534 #ifdef SMP 535 if (smp_cpus > 1) { 536 int count = BUSY_SPINS; 537 if (count > 0) { 538 umtxq_unlock(key); 539 while (uc->uc_busy && --count > 0) 540 cpu_spinwait(); 541 umtxq_lock(key); 542 } 543 } 544 #endif 545 while (uc->uc_busy) { 546 uc->uc_waiters++; 547 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); 548 uc->uc_waiters--; 549 } 550 } 551 uc->uc_busy = 1; 552 } 553 554 /* 555 * Unbusy a chain. 556 */ 557 static inline void 558 umtxq_unbusy(struct umtx_key *key) 559 { 560 struct umtxq_chain *uc; 561 562 uc = umtxq_getchain(key); 563 mtx_assert(&uc->uc_lock, MA_OWNED); 564 KASSERT(uc->uc_busy != 0, ("not busy")); 565 uc->uc_busy = 0; 566 if (uc->uc_waiters) 567 wakeup_one(uc); 568 } 569 570 static inline void 571 umtxq_unbusy_unlocked(struct umtx_key *key) 572 { 573 574 umtxq_lock(key); 575 umtxq_unbusy(key); 576 umtxq_unlock(key); 577 } 578 579 static struct umtxq_queue * 580 umtxq_queue_lookup(struct umtx_key *key, int q) 581 { 582 struct umtxq_queue *uh; 583 struct umtxq_chain *uc; 584 585 uc = umtxq_getchain(key); 586 UMTXQ_LOCKED_ASSERT(uc); 587 LIST_FOREACH(uh, &uc->uc_queue[q], link) { 588 if (umtx_key_match(&uh->key, key)) 589 return (uh); 590 } 591 592 return (NULL); 593 } 594 595 static inline void 596 umtxq_insert_queue(struct umtx_q *uq, int q) 597 { 598 struct umtxq_queue *uh; 599 struct umtxq_chain *uc; 600 601 uc = umtxq_getchain(&uq->uq_key); 602 UMTXQ_LOCKED_ASSERT(uc); 603 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue")); 604 uh = umtxq_queue_lookup(&uq->uq_key, q); 605 if (uh != NULL) { 606 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link); 607 } else { 608 uh = uq->uq_spare_queue; 609 uh->key = uq->uq_key; 610 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link); 611 #ifdef UMTX_PROFILING 612 uc->length++; 613 if (uc->length > uc->max_length) { 614 uc->max_length = uc->length; 615 if (uc->max_length > max_length) 616 max_length = uc->max_length; 617 } 618 #endif 619 } 620 uq->uq_spare_queue = NULL; 621 622 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link); 623 uh->length++; 624 uq->uq_flags |= UQF_UMTXQ; 625 uq->uq_cur_queue = uh; 626 return; 627 } 628 629 static inline void 630 umtxq_remove_queue(struct umtx_q *uq, int q) 631 { 632 struct umtxq_chain *uc; 633 struct umtxq_queue *uh; 634 635 uc = umtxq_getchain(&uq->uq_key); 636 UMTXQ_LOCKED_ASSERT(uc); 637 if (uq->uq_flags & UQF_UMTXQ) { 638 uh = uq->uq_cur_queue; 639 TAILQ_REMOVE(&uh->head, uq, uq_link); 640 uh->length--; 641 uq->uq_flags &= ~UQF_UMTXQ; 642 if (TAILQ_EMPTY(&uh->head)) { 643 KASSERT(uh->length == 0, 644 ("inconsistent umtxq_queue length")); 645 #ifdef UMTX_PROFILING 646 uc->length--; 647 #endif 648 LIST_REMOVE(uh, link); 649 } else { 650 uh = LIST_FIRST(&uc->uc_spare_queue); 651 KASSERT(uh != NULL, ("uc_spare_queue is empty")); 652 LIST_REMOVE(uh, link); 653 } 654 uq->uq_spare_queue = uh; 655 uq->uq_cur_queue = NULL; 656 } 657 } 658 659 /* 660 * Check if there are multiple waiters 661 */ 662 static int 663 umtxq_count(struct umtx_key *key) 664 { 665 struct umtxq_queue *uh; 666 667 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key)); 668 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); 669 if (uh != NULL) 670 return (uh->length); 671 return (0); 672 } 673 674 /* 675 * Check if there are multiple PI waiters and returns first 676 * waiter. 677 */ 678 static int 679 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) 680 { 681 struct umtxq_queue *uh; 682 683 *first = NULL; 684 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key)); 685 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); 686 if (uh != NULL) { 687 *first = TAILQ_FIRST(&uh->head); 688 return (uh->length); 689 } 690 return (0); 691 } 692 693 /* 694 * Check for possible stops and suspensions while executing a umtx 695 * locking operation. 696 * 697 * The sleep argument controls whether the function can handle a stop 698 * request itself or it should return ERESTART and the request is 699 * proceed at the kernel/user boundary in ast. 700 * 701 * Typically, when retrying due to casueword(9) failure (rv == 1), we 702 * should handle the stop requests there, with exception of cases when 703 * the thread busied the umtx key, or when functions return 704 * immediately if umtxq_check_susp() returned non-zero. On the other 705 * hand, retrying the whole lock operation, we better not stop there 706 * but delegate the handling to ast. 707 * 708 * If the request is for thread termination P_SINGLE_EXIT, we cannot 709 * handle it at all, and simply return EINTR. 710 */ 711 static int 712 umtxq_check_susp(struct thread *td, bool sleep) 713 { 714 struct proc *p; 715 int error; 716 717 /* 718 * The check for TDF_NEEDSUSPCHK is racy, but it is enough to 719 * eventually break the lockstep loop. 720 */ 721 if ((td->td_flags & TDF_NEEDSUSPCHK) == 0) 722 return (0); 723 error = 0; 724 p = td->td_proc; 725 PROC_LOCK(p); 726 if (p->p_flag & P_SINGLE_EXIT) 727 error = EINTR; 728 else if (P_SHOULDSTOP(p) || 729 ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) 730 error = sleep ? thread_suspend_check(0) : ERESTART; 731 PROC_UNLOCK(p); 732 return (error); 733 } 734 735 /* 736 * Wake up threads waiting on an userland object. 737 */ 738 739 static int 740 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q) 741 { 742 struct umtxq_queue *uh; 743 struct umtx_q *uq; 744 int ret; 745 746 ret = 0; 747 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key)); 748 uh = umtxq_queue_lookup(key, q); 749 if (uh != NULL) { 750 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) { 751 umtxq_remove_queue(uq, q); 752 wakeup(uq); 753 if (++ret >= n_wake) 754 return (ret); 755 } 756 } 757 return (ret); 758 } 759 760 761 /* 762 * Wake up specified thread. 763 */ 764 static inline void 765 umtxq_signal_thread(struct umtx_q *uq) 766 { 767 768 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key)); 769 umtxq_remove(uq); 770 wakeup(uq); 771 } 772 773 static inline int 774 tstohz(const struct timespec *tsp) 775 { 776 struct timeval tv; 777 778 TIMESPEC_TO_TIMEVAL(&tv, tsp); 779 return tvtohz(&tv); 780 } 781 782 static void 783 abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute, 784 const struct timespec *timeout) 785 { 786 787 timo->clockid = clockid; 788 if (!absolute) { 789 timo->is_abs_real = false; 790 abs_timeout_update(timo); 791 timespecadd(&timo->cur, timeout, &timo->end); 792 } else { 793 timo->end = *timeout; 794 timo->is_abs_real = clockid == CLOCK_REALTIME || 795 clockid == CLOCK_REALTIME_FAST || 796 clockid == CLOCK_REALTIME_PRECISE; 797 /* 798 * If is_abs_real, umtxq_sleep will read the clock 799 * after setting td_rtcgen; otherwise, read it here. 800 */ 801 if (!timo->is_abs_real) { 802 abs_timeout_update(timo); 803 } 804 } 805 } 806 807 static void 808 abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime) 809 { 810 811 abs_timeout_init(timo, umtxtime->_clockid, 812 (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); 813 } 814 815 static inline void 816 abs_timeout_update(struct abs_timeout *timo) 817 { 818 819 kern_clock_gettime(curthread, timo->clockid, &timo->cur); 820 } 821 822 static int 823 abs_timeout_gethz(struct abs_timeout *timo) 824 { 825 struct timespec tts; 826 827 if (timespeccmp(&timo->end, &timo->cur, <=)) 828 return (-1); 829 timespecsub(&timo->end, &timo->cur, &tts); 830 return (tstohz(&tts)); 831 } 832 833 static uint32_t 834 umtx_unlock_val(uint32_t flags, bool rb) 835 { 836 837 if (rb) 838 return (UMUTEX_RB_OWNERDEAD); 839 else if ((flags & UMUTEX_NONCONSISTENT) != 0) 840 return (UMUTEX_RB_NOTRECOV); 841 else 842 return (UMUTEX_UNOWNED); 843 844 } 845 846 /* 847 * Put thread into sleep state, before sleeping, check if 848 * thread was removed from umtx queue. 849 */ 850 static inline int 851 umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime) 852 { 853 struct umtxq_chain *uc; 854 int error, timo; 855 856 if (abstime != NULL && abstime->is_abs_real) { 857 curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation); 858 abs_timeout_update(abstime); 859 } 860 861 uc = umtxq_getchain(&uq->uq_key); 862 UMTXQ_LOCKED_ASSERT(uc); 863 for (;;) { 864 if (!(uq->uq_flags & UQF_UMTXQ)) { 865 error = 0; 866 break; 867 } 868 if (abstime != NULL) { 869 timo = abs_timeout_gethz(abstime); 870 if (timo < 0) { 871 error = ETIMEDOUT; 872 break; 873 } 874 } else 875 timo = 0; 876 error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo); 877 if (error == EINTR || error == ERESTART) { 878 umtxq_lock(&uq->uq_key); 879 break; 880 } 881 if (abstime != NULL) { 882 if (abstime->is_abs_real) 883 curthread->td_rtcgen = 884 atomic_load_acq_int(&rtc_generation); 885 abs_timeout_update(abstime); 886 } 887 umtxq_lock(&uq->uq_key); 888 } 889 890 curthread->td_rtcgen = 0; 891 return (error); 892 } 893 894 /* 895 * Convert userspace address into unique logical address. 896 */ 897 int 898 umtx_key_get(const void *addr, int type, int share, struct umtx_key *key) 899 { 900 struct thread *td = curthread; 901 vm_map_t map; 902 vm_map_entry_t entry; 903 vm_pindex_t pindex; 904 vm_prot_t prot; 905 boolean_t wired; 906 907 key->type = type; 908 if (share == THREAD_SHARE) { 909 key->shared = 0; 910 key->info.private.vs = td->td_proc->p_vmspace; 911 key->info.private.addr = (uintptr_t)addr; 912 } else { 913 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE); 914 map = &td->td_proc->p_vmspace->vm_map; 915 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE, 916 &entry, &key->info.shared.object, &pindex, &prot, 917 &wired) != KERN_SUCCESS) { 918 return (EFAULT); 919 } 920 921 if ((share == PROCESS_SHARE) || 922 (share == AUTO_SHARE && 923 VM_INHERIT_SHARE == entry->inheritance)) { 924 key->shared = 1; 925 key->info.shared.offset = (vm_offset_t)addr - 926 entry->start + entry->offset; 927 vm_object_reference(key->info.shared.object); 928 } else { 929 key->shared = 0; 930 key->info.private.vs = td->td_proc->p_vmspace; 931 key->info.private.addr = (uintptr_t)addr; 932 } 933 vm_map_lookup_done(map, entry); 934 } 935 936 umtxq_hash(key); 937 return (0); 938 } 939 940 /* 941 * Release key. 942 */ 943 void 944 umtx_key_release(struct umtx_key *key) 945 { 946 if (key->shared) 947 vm_object_deallocate(key->info.shared.object); 948 } 949 950 /* 951 * Fetch and compare value, sleep on the address if value is not changed. 952 */ 953 static int 954 do_wait(struct thread *td, void *addr, u_long id, 955 struct _umtx_time *timeout, int compat32, int is_private) 956 { 957 struct abs_timeout timo; 958 struct umtx_q *uq; 959 u_long tmp; 960 uint32_t tmp32; 961 int error = 0; 962 963 uq = td->td_umtxq; 964 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, 965 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) 966 return (error); 967 968 if (timeout != NULL) 969 abs_timeout_init2(&timo, timeout); 970 971 umtxq_lock(&uq->uq_key); 972 umtxq_insert(uq); 973 umtxq_unlock(&uq->uq_key); 974 if (compat32 == 0) { 975 error = fueword(addr, &tmp); 976 if (error != 0) 977 error = EFAULT; 978 } else { 979 error = fueword32(addr, &tmp32); 980 if (error == 0) 981 tmp = tmp32; 982 else 983 error = EFAULT; 984 } 985 umtxq_lock(&uq->uq_key); 986 if (error == 0) { 987 if (tmp == id) 988 error = umtxq_sleep(uq, "uwait", timeout == NULL ? 989 NULL : &timo); 990 if ((uq->uq_flags & UQF_UMTXQ) == 0) 991 error = 0; 992 else 993 umtxq_remove(uq); 994 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) { 995 umtxq_remove(uq); 996 } 997 umtxq_unlock(&uq->uq_key); 998 umtx_key_release(&uq->uq_key); 999 if (error == ERESTART) 1000 error = EINTR; 1001 return (error); 1002 } 1003 1004 /* 1005 * Wake up threads sleeping on the specified address. 1006 */ 1007 int 1008 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) 1009 { 1010 struct umtx_key key; 1011 int ret; 1012 1013 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, 1014 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) 1015 return (ret); 1016 umtxq_lock(&key); 1017 umtxq_signal(&key, n_wake); 1018 umtxq_unlock(&key); 1019 umtx_key_release(&key); 1020 return (0); 1021 } 1022 1023 /* 1024 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1025 */ 1026 static int 1027 do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, 1028 struct _umtx_time *timeout, int mode) 1029 { 1030 struct abs_timeout timo; 1031 struct umtx_q *uq; 1032 uint32_t owner, old, id; 1033 int error, rv; 1034 1035 id = td->td_tid; 1036 uq = td->td_umtxq; 1037 error = 0; 1038 if (timeout != NULL) 1039 abs_timeout_init2(&timo, timeout); 1040 1041 /* 1042 * Care must be exercised when dealing with umtx structure. It 1043 * can fault on any access. 1044 */ 1045 for (;;) { 1046 rv = fueword32(&m->m_owner, &owner); 1047 if (rv == -1) 1048 return (EFAULT); 1049 if (mode == _UMUTEX_WAIT) { 1050 if (owner == UMUTEX_UNOWNED || 1051 owner == UMUTEX_CONTESTED || 1052 owner == UMUTEX_RB_OWNERDEAD || 1053 owner == UMUTEX_RB_NOTRECOV) 1054 return (0); 1055 } else { 1056 /* 1057 * Robust mutex terminated. Kernel duty is to 1058 * return EOWNERDEAD to the userspace. The 1059 * umutex.m_flags UMUTEX_NONCONSISTENT is set 1060 * by the common userspace code. 1061 */ 1062 if (owner == UMUTEX_RB_OWNERDEAD) { 1063 rv = casueword32(&m->m_owner, 1064 UMUTEX_RB_OWNERDEAD, &owner, 1065 id | UMUTEX_CONTESTED); 1066 if (rv == -1) 1067 return (EFAULT); 1068 if (rv == 0) { 1069 MPASS(owner == UMUTEX_RB_OWNERDEAD); 1070 return (EOWNERDEAD); /* success */ 1071 } 1072 MPASS(rv == 1); 1073 rv = umtxq_check_susp(td, false); 1074 if (rv != 0) 1075 return (rv); 1076 continue; 1077 } 1078 if (owner == UMUTEX_RB_NOTRECOV) 1079 return (ENOTRECOVERABLE); 1080 1081 /* 1082 * Try the uncontested case. This should be 1083 * done in userland. 1084 */ 1085 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, 1086 &owner, id); 1087 /* The address was invalid. */ 1088 if (rv == -1) 1089 return (EFAULT); 1090 1091 /* The acquire succeeded. */ 1092 if (rv == 0) { 1093 MPASS(owner == UMUTEX_UNOWNED); 1094 return (0); 1095 } 1096 1097 /* 1098 * If no one owns it but it is contested try 1099 * to acquire it. 1100 */ 1101 MPASS(rv == 1); 1102 if (owner == UMUTEX_CONTESTED) { 1103 rv = casueword32(&m->m_owner, 1104 UMUTEX_CONTESTED, &owner, 1105 id | UMUTEX_CONTESTED); 1106 /* The address was invalid. */ 1107 if (rv == -1) 1108 return (EFAULT); 1109 if (rv == 0) { 1110 MPASS(owner == UMUTEX_CONTESTED); 1111 return (0); 1112 } 1113 if (rv == 1) { 1114 rv = umtxq_check_susp(td, false); 1115 if (rv != 0) 1116 return (rv); 1117 } 1118 1119 /* 1120 * If this failed the lock has 1121 * changed, restart. 1122 */ 1123 continue; 1124 } 1125 1126 /* rv == 1 but not contested, likely store failure */ 1127 rv = umtxq_check_susp(td, false); 1128 if (rv != 0) 1129 return (rv); 1130 } 1131 1132 if (mode == _UMUTEX_TRY) 1133 return (EBUSY); 1134 1135 /* 1136 * If we caught a signal, we have retried and now 1137 * exit immediately. 1138 */ 1139 if (error != 0) 1140 return (error); 1141 1142 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, 1143 GET_SHARE(flags), &uq->uq_key)) != 0) 1144 return (error); 1145 1146 umtxq_lock(&uq->uq_key); 1147 umtxq_busy(&uq->uq_key); 1148 umtxq_insert(uq); 1149 umtxq_unlock(&uq->uq_key); 1150 1151 /* 1152 * Set the contested bit so that a release in user space 1153 * knows to use the system call for unlock. If this fails 1154 * either some one else has acquired the lock or it has been 1155 * released. 1156 */ 1157 rv = casueword32(&m->m_owner, owner, &old, 1158 owner | UMUTEX_CONTESTED); 1159 1160 /* The address was invalid or casueword failed to store. */ 1161 if (rv == -1 || rv == 1) { 1162 umtxq_lock(&uq->uq_key); 1163 umtxq_remove(uq); 1164 umtxq_unbusy(&uq->uq_key); 1165 umtxq_unlock(&uq->uq_key); 1166 umtx_key_release(&uq->uq_key); 1167 if (rv == -1) 1168 return (EFAULT); 1169 if (rv == 1) { 1170 rv = umtxq_check_susp(td, false); 1171 if (rv != 0) 1172 return (rv); 1173 } 1174 continue; 1175 } 1176 1177 /* 1178 * We set the contested bit, sleep. Otherwise the lock changed 1179 * and we need to retry or we lost a race to the thread 1180 * unlocking the umtx. 1181 */ 1182 umtxq_lock(&uq->uq_key); 1183 umtxq_unbusy(&uq->uq_key); 1184 MPASS(old == owner); 1185 error = umtxq_sleep(uq, "umtxn", timeout == NULL ? 1186 NULL : &timo); 1187 umtxq_remove(uq); 1188 umtxq_unlock(&uq->uq_key); 1189 umtx_key_release(&uq->uq_key); 1190 1191 if (error == 0) 1192 error = umtxq_check_susp(td, false); 1193 } 1194 1195 return (0); 1196 } 1197 1198 /* 1199 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. 1200 */ 1201 static int 1202 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb) 1203 { 1204 struct umtx_key key; 1205 uint32_t owner, old, id, newlock; 1206 int error, count; 1207 1208 id = td->td_tid; 1209 1210 again: 1211 /* 1212 * Make sure we own this mtx. 1213 */ 1214 error = fueword32(&m->m_owner, &owner); 1215 if (error == -1) 1216 return (EFAULT); 1217 1218 if ((owner & ~UMUTEX_CONTESTED) != id) 1219 return (EPERM); 1220 1221 newlock = umtx_unlock_val(flags, rb); 1222 if ((owner & UMUTEX_CONTESTED) == 0) { 1223 error = casueword32(&m->m_owner, owner, &old, newlock); 1224 if (error == -1) 1225 return (EFAULT); 1226 if (error == 1) { 1227 error = umtxq_check_susp(td, false); 1228 if (error != 0) 1229 return (error); 1230 goto again; 1231 } 1232 MPASS(old == owner); 1233 return (0); 1234 } 1235 1236 /* We should only ever be in here for contested locks */ 1237 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1238 &key)) != 0) 1239 return (error); 1240 1241 umtxq_lock(&key); 1242 umtxq_busy(&key); 1243 count = umtxq_count(&key); 1244 umtxq_unlock(&key); 1245 1246 /* 1247 * When unlocking the umtx, it must be marked as unowned if 1248 * there is zero or one thread only waiting for it. 1249 * Otherwise, it must be marked as contested. 1250 */ 1251 if (count > 1) 1252 newlock |= UMUTEX_CONTESTED; 1253 error = casueword32(&m->m_owner, owner, &old, newlock); 1254 umtxq_lock(&key); 1255 umtxq_signal(&key, 1); 1256 umtxq_unbusy(&key); 1257 umtxq_unlock(&key); 1258 umtx_key_release(&key); 1259 if (error == -1) 1260 return (EFAULT); 1261 if (error == 1) { 1262 if (old != owner) 1263 return (EINVAL); 1264 error = umtxq_check_susp(td, false); 1265 if (error != 0) 1266 return (error); 1267 goto again; 1268 } 1269 return (0); 1270 } 1271 1272 /* 1273 * Check if the mutex is available and wake up a waiter, 1274 * only for simple mutex. 1275 */ 1276 static int 1277 do_wake_umutex(struct thread *td, struct umutex *m) 1278 { 1279 struct umtx_key key; 1280 uint32_t owner; 1281 uint32_t flags; 1282 int error; 1283 int count; 1284 1285 again: 1286 error = fueword32(&m->m_owner, &owner); 1287 if (error == -1) 1288 return (EFAULT); 1289 1290 if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD && 1291 owner != UMUTEX_RB_NOTRECOV) 1292 return (0); 1293 1294 error = fueword32(&m->m_flags, &flags); 1295 if (error == -1) 1296 return (EFAULT); 1297 1298 /* We should only ever be in here for contested locks */ 1299 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1300 &key)) != 0) 1301 return (error); 1302 1303 umtxq_lock(&key); 1304 umtxq_busy(&key); 1305 count = umtxq_count(&key); 1306 umtxq_unlock(&key); 1307 1308 if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD && 1309 owner != UMUTEX_RB_NOTRECOV) { 1310 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, 1311 UMUTEX_UNOWNED); 1312 if (error == -1) { 1313 error = EFAULT; 1314 } else if (error == 1) { 1315 umtxq_lock(&key); 1316 umtxq_unbusy(&key); 1317 umtxq_unlock(&key); 1318 umtx_key_release(&key); 1319 error = umtxq_check_susp(td, false); 1320 if (error != 0) 1321 return (error); 1322 goto again; 1323 } 1324 } 1325 1326 umtxq_lock(&key); 1327 if (error == 0 && count != 0) { 1328 MPASS((owner & ~UMUTEX_CONTESTED) == 0 || 1329 owner == UMUTEX_RB_OWNERDEAD || 1330 owner == UMUTEX_RB_NOTRECOV); 1331 umtxq_signal(&key, 1); 1332 } 1333 umtxq_unbusy(&key); 1334 umtxq_unlock(&key); 1335 umtx_key_release(&key); 1336 return (error); 1337 } 1338 1339 /* 1340 * Check if the mutex has waiters and tries to fix contention bit. 1341 */ 1342 static int 1343 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) 1344 { 1345 struct umtx_key key; 1346 uint32_t owner, old; 1347 int type; 1348 int error; 1349 int count; 1350 1351 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT | 1352 UMUTEX_ROBUST)) { 1353 case 0: 1354 case UMUTEX_ROBUST: 1355 type = TYPE_NORMAL_UMUTEX; 1356 break; 1357 case UMUTEX_PRIO_INHERIT: 1358 type = TYPE_PI_UMUTEX; 1359 break; 1360 case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST): 1361 type = TYPE_PI_ROBUST_UMUTEX; 1362 break; 1363 case UMUTEX_PRIO_PROTECT: 1364 type = TYPE_PP_UMUTEX; 1365 break; 1366 case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST): 1367 type = TYPE_PP_ROBUST_UMUTEX; 1368 break; 1369 default: 1370 return (EINVAL); 1371 } 1372 if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0) 1373 return (error); 1374 1375 owner = 0; 1376 umtxq_lock(&key); 1377 umtxq_busy(&key); 1378 count = umtxq_count(&key); 1379 umtxq_unlock(&key); 1380 1381 error = fueword32(&m->m_owner, &owner); 1382 if (error == -1) 1383 error = EFAULT; 1384 1385 /* 1386 * Only repair contention bit if there is a waiter, this means 1387 * the mutex is still being referenced by userland code, 1388 * otherwise don't update any memory. 1389 */ 1390 while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 && 1391 (count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) { 1392 error = casueword32(&m->m_owner, owner, &old, 1393 owner | UMUTEX_CONTESTED); 1394 if (error == -1) { 1395 error = EFAULT; 1396 break; 1397 } 1398 if (error == 0) { 1399 MPASS(old == owner); 1400 break; 1401 } 1402 owner = old; 1403 error = umtxq_check_susp(td, false); 1404 } 1405 1406 umtxq_lock(&key); 1407 if (error == EFAULT) { 1408 umtxq_signal(&key, INT_MAX); 1409 } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || 1410 owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) 1411 umtxq_signal(&key, 1); 1412 umtxq_unbusy(&key); 1413 umtxq_unlock(&key); 1414 umtx_key_release(&key); 1415 return (error); 1416 } 1417 1418 static inline struct umtx_pi * 1419 umtx_pi_alloc(int flags) 1420 { 1421 struct umtx_pi *pi; 1422 1423 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags); 1424 TAILQ_INIT(&pi->pi_blocked); 1425 atomic_add_int(&umtx_pi_allocated, 1); 1426 return (pi); 1427 } 1428 1429 static inline void 1430 umtx_pi_free(struct umtx_pi *pi) 1431 { 1432 uma_zfree(umtx_pi_zone, pi); 1433 atomic_add_int(&umtx_pi_allocated, -1); 1434 } 1435 1436 /* 1437 * Adjust the thread's position on a pi_state after its priority has been 1438 * changed. 1439 */ 1440 static int 1441 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) 1442 { 1443 struct umtx_q *uq, *uq1, *uq2; 1444 struct thread *td1; 1445 1446 mtx_assert(&umtx_lock, MA_OWNED); 1447 if (pi == NULL) 1448 return (0); 1449 1450 uq = td->td_umtxq; 1451 1452 /* 1453 * Check if the thread needs to be moved on the blocked chain. 1454 * It needs to be moved if either its priority is lower than 1455 * the previous thread or higher than the next thread. 1456 */ 1457 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq); 1458 uq2 = TAILQ_NEXT(uq, uq_lockq); 1459 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || 1460 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { 1461 /* 1462 * Remove thread from blocked chain and determine where 1463 * it should be moved to. 1464 */ 1465 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1466 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1467 td1 = uq1->uq_thread; 1468 MPASS(td1->td_proc->p_magic == P_MAGIC); 1469 if (UPRI(td1) > UPRI(td)) 1470 break; 1471 } 1472 1473 if (uq1 == NULL) 1474 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1475 else 1476 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1477 } 1478 return (1); 1479 } 1480 1481 static struct umtx_pi * 1482 umtx_pi_next(struct umtx_pi *pi) 1483 { 1484 struct umtx_q *uq_owner; 1485 1486 if (pi->pi_owner == NULL) 1487 return (NULL); 1488 uq_owner = pi->pi_owner->td_umtxq; 1489 if (uq_owner == NULL) 1490 return (NULL); 1491 return (uq_owner->uq_pi_blocked); 1492 } 1493 1494 /* 1495 * Floyd's Cycle-Finding Algorithm. 1496 */ 1497 static bool 1498 umtx_pi_check_loop(struct umtx_pi *pi) 1499 { 1500 struct umtx_pi *pi1; /* fast iterator */ 1501 1502 mtx_assert(&umtx_lock, MA_OWNED); 1503 if (pi == NULL) 1504 return (false); 1505 pi1 = pi; 1506 for (;;) { 1507 pi = umtx_pi_next(pi); 1508 if (pi == NULL) 1509 break; 1510 pi1 = umtx_pi_next(pi1); 1511 if (pi1 == NULL) 1512 break; 1513 pi1 = umtx_pi_next(pi1); 1514 if (pi1 == NULL) 1515 break; 1516 if (pi == pi1) 1517 return (true); 1518 } 1519 return (false); 1520 } 1521 1522 /* 1523 * Propagate priority when a thread is blocked on POSIX 1524 * PI mutex. 1525 */ 1526 static void 1527 umtx_propagate_priority(struct thread *td) 1528 { 1529 struct umtx_q *uq; 1530 struct umtx_pi *pi; 1531 int pri; 1532 1533 mtx_assert(&umtx_lock, MA_OWNED); 1534 pri = UPRI(td); 1535 uq = td->td_umtxq; 1536 pi = uq->uq_pi_blocked; 1537 if (pi == NULL) 1538 return; 1539 if (umtx_pi_check_loop(pi)) 1540 return; 1541 1542 for (;;) { 1543 td = pi->pi_owner; 1544 if (td == NULL || td == curthread) 1545 return; 1546 1547 MPASS(td->td_proc != NULL); 1548 MPASS(td->td_proc->p_magic == P_MAGIC); 1549 1550 thread_lock(td); 1551 if (td->td_lend_user_pri > pri) 1552 sched_lend_user_prio(td, pri); 1553 else { 1554 thread_unlock(td); 1555 break; 1556 } 1557 thread_unlock(td); 1558 1559 /* 1560 * Pick up the lock that td is blocked on. 1561 */ 1562 uq = td->td_umtxq; 1563 pi = uq->uq_pi_blocked; 1564 if (pi == NULL) 1565 break; 1566 /* Resort td on the list if needed. */ 1567 umtx_pi_adjust_thread(pi, td); 1568 } 1569 } 1570 1571 /* 1572 * Unpropagate priority for a PI mutex when a thread blocked on 1573 * it is interrupted by signal or resumed by others. 1574 */ 1575 static void 1576 umtx_repropagate_priority(struct umtx_pi *pi) 1577 { 1578 struct umtx_q *uq, *uq_owner; 1579 struct umtx_pi *pi2; 1580 int pri; 1581 1582 mtx_assert(&umtx_lock, MA_OWNED); 1583 1584 if (umtx_pi_check_loop(pi)) 1585 return; 1586 while (pi != NULL && pi->pi_owner != NULL) { 1587 pri = PRI_MAX; 1588 uq_owner = pi->pi_owner->td_umtxq; 1589 1590 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { 1591 uq = TAILQ_FIRST(&pi2->pi_blocked); 1592 if (uq != NULL) { 1593 if (pri > UPRI(uq->uq_thread)) 1594 pri = UPRI(uq->uq_thread); 1595 } 1596 } 1597 1598 if (pri > uq_owner->uq_inherited_pri) 1599 pri = uq_owner->uq_inherited_pri; 1600 thread_lock(pi->pi_owner); 1601 sched_lend_user_prio(pi->pi_owner, pri); 1602 thread_unlock(pi->pi_owner); 1603 if ((pi = uq_owner->uq_pi_blocked) != NULL) 1604 umtx_pi_adjust_thread(pi, uq_owner->uq_thread); 1605 } 1606 } 1607 1608 /* 1609 * Insert a PI mutex into owned list. 1610 */ 1611 static void 1612 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) 1613 { 1614 struct umtx_q *uq_owner; 1615 1616 uq_owner = owner->td_umtxq; 1617 mtx_assert(&umtx_lock, MA_OWNED); 1618 MPASS(pi->pi_owner == NULL); 1619 pi->pi_owner = owner; 1620 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); 1621 } 1622 1623 1624 /* 1625 * Disown a PI mutex, and remove it from the owned list. 1626 */ 1627 static void 1628 umtx_pi_disown(struct umtx_pi *pi) 1629 { 1630 1631 mtx_assert(&umtx_lock, MA_OWNED); 1632 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link); 1633 pi->pi_owner = NULL; 1634 } 1635 1636 /* 1637 * Claim ownership of a PI mutex. 1638 */ 1639 static int 1640 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) 1641 { 1642 struct umtx_q *uq; 1643 int pri; 1644 1645 mtx_lock(&umtx_lock); 1646 if (pi->pi_owner == owner) { 1647 mtx_unlock(&umtx_lock); 1648 return (0); 1649 } 1650 1651 if (pi->pi_owner != NULL) { 1652 /* 1653 * userland may have already messed the mutex, sigh. 1654 */ 1655 mtx_unlock(&umtx_lock); 1656 return (EPERM); 1657 } 1658 umtx_pi_setowner(pi, owner); 1659 uq = TAILQ_FIRST(&pi->pi_blocked); 1660 if (uq != NULL) { 1661 pri = UPRI(uq->uq_thread); 1662 thread_lock(owner); 1663 if (pri < UPRI(owner)) 1664 sched_lend_user_prio(owner, pri); 1665 thread_unlock(owner); 1666 } 1667 mtx_unlock(&umtx_lock); 1668 return (0); 1669 } 1670 1671 /* 1672 * Adjust a thread's order position in its blocked PI mutex, 1673 * this may result new priority propagating process. 1674 */ 1675 void 1676 umtx_pi_adjust(struct thread *td, u_char oldpri) 1677 { 1678 struct umtx_q *uq; 1679 struct umtx_pi *pi; 1680 1681 uq = td->td_umtxq; 1682 mtx_lock(&umtx_lock); 1683 /* 1684 * Pick up the lock that td is blocked on. 1685 */ 1686 pi = uq->uq_pi_blocked; 1687 if (pi != NULL) { 1688 umtx_pi_adjust_thread(pi, td); 1689 umtx_repropagate_priority(pi); 1690 } 1691 mtx_unlock(&umtx_lock); 1692 } 1693 1694 /* 1695 * Sleep on a PI mutex. 1696 */ 1697 static int 1698 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner, 1699 const char *wmesg, struct abs_timeout *timo, bool shared) 1700 { 1701 struct thread *td, *td1; 1702 struct umtx_q *uq1; 1703 int error, pri; 1704 #ifdef INVARIANTS 1705 struct umtxq_chain *uc; 1706 1707 uc = umtxq_getchain(&pi->pi_key); 1708 #endif 1709 error = 0; 1710 td = uq->uq_thread; 1711 KASSERT(td == curthread, ("inconsistent uq_thread")); 1712 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key)); 1713 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); 1714 umtxq_insert(uq); 1715 mtx_lock(&umtx_lock); 1716 if (pi->pi_owner == NULL) { 1717 mtx_unlock(&umtx_lock); 1718 td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid); 1719 mtx_lock(&umtx_lock); 1720 if (td1 != NULL) { 1721 if (pi->pi_owner == NULL) 1722 umtx_pi_setowner(pi, td1); 1723 PROC_UNLOCK(td1->td_proc); 1724 } 1725 } 1726 1727 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1728 pri = UPRI(uq1->uq_thread); 1729 if (pri > UPRI(td)) 1730 break; 1731 } 1732 1733 if (uq1 != NULL) 1734 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1735 else 1736 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1737 1738 uq->uq_pi_blocked = pi; 1739 thread_lock(td); 1740 td->td_flags |= TDF_UPIBLOCKED; 1741 thread_unlock(td); 1742 umtx_propagate_priority(td); 1743 mtx_unlock(&umtx_lock); 1744 umtxq_unbusy(&uq->uq_key); 1745 1746 error = umtxq_sleep(uq, wmesg, timo); 1747 umtxq_remove(uq); 1748 1749 mtx_lock(&umtx_lock); 1750 uq->uq_pi_blocked = NULL; 1751 thread_lock(td); 1752 td->td_flags &= ~TDF_UPIBLOCKED; 1753 thread_unlock(td); 1754 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1755 umtx_repropagate_priority(pi); 1756 mtx_unlock(&umtx_lock); 1757 umtxq_unlock(&uq->uq_key); 1758 1759 return (error); 1760 } 1761 1762 /* 1763 * Add reference count for a PI mutex. 1764 */ 1765 static void 1766 umtx_pi_ref(struct umtx_pi *pi) 1767 { 1768 1769 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key)); 1770 pi->pi_refcount++; 1771 } 1772 1773 /* 1774 * Decrease reference count for a PI mutex, if the counter 1775 * is decreased to zero, its memory space is freed. 1776 */ 1777 static void 1778 umtx_pi_unref(struct umtx_pi *pi) 1779 { 1780 struct umtxq_chain *uc; 1781 1782 uc = umtxq_getchain(&pi->pi_key); 1783 UMTXQ_LOCKED_ASSERT(uc); 1784 KASSERT(pi->pi_refcount > 0, ("invalid reference count")); 1785 if (--pi->pi_refcount == 0) { 1786 mtx_lock(&umtx_lock); 1787 if (pi->pi_owner != NULL) 1788 umtx_pi_disown(pi); 1789 KASSERT(TAILQ_EMPTY(&pi->pi_blocked), 1790 ("blocked queue not empty")); 1791 mtx_unlock(&umtx_lock); 1792 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); 1793 umtx_pi_free(pi); 1794 } 1795 } 1796 1797 /* 1798 * Find a PI mutex in hash table. 1799 */ 1800 static struct umtx_pi * 1801 umtx_pi_lookup(struct umtx_key *key) 1802 { 1803 struct umtxq_chain *uc; 1804 struct umtx_pi *pi; 1805 1806 uc = umtxq_getchain(key); 1807 UMTXQ_LOCKED_ASSERT(uc); 1808 1809 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { 1810 if (umtx_key_match(&pi->pi_key, key)) { 1811 return (pi); 1812 } 1813 } 1814 return (NULL); 1815 } 1816 1817 /* 1818 * Insert a PI mutex into hash table. 1819 */ 1820 static inline void 1821 umtx_pi_insert(struct umtx_pi *pi) 1822 { 1823 struct umtxq_chain *uc; 1824 1825 uc = umtxq_getchain(&pi->pi_key); 1826 UMTXQ_LOCKED_ASSERT(uc); 1827 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); 1828 } 1829 1830 /* 1831 * Lock a PI mutex. 1832 */ 1833 static int 1834 do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, 1835 struct _umtx_time *timeout, int try) 1836 { 1837 struct abs_timeout timo; 1838 struct umtx_q *uq; 1839 struct umtx_pi *pi, *new_pi; 1840 uint32_t id, old_owner, owner, old; 1841 int error, rv; 1842 1843 id = td->td_tid; 1844 uq = td->td_umtxq; 1845 1846 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 1847 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), 1848 &uq->uq_key)) != 0) 1849 return (error); 1850 1851 if (timeout != NULL) 1852 abs_timeout_init2(&timo, timeout); 1853 1854 umtxq_lock(&uq->uq_key); 1855 pi = umtx_pi_lookup(&uq->uq_key); 1856 if (pi == NULL) { 1857 new_pi = umtx_pi_alloc(M_NOWAIT); 1858 if (new_pi == NULL) { 1859 umtxq_unlock(&uq->uq_key); 1860 new_pi = umtx_pi_alloc(M_WAITOK); 1861 umtxq_lock(&uq->uq_key); 1862 pi = umtx_pi_lookup(&uq->uq_key); 1863 if (pi != NULL) { 1864 umtx_pi_free(new_pi); 1865 new_pi = NULL; 1866 } 1867 } 1868 if (new_pi != NULL) { 1869 new_pi->pi_key = uq->uq_key; 1870 umtx_pi_insert(new_pi); 1871 pi = new_pi; 1872 } 1873 } 1874 umtx_pi_ref(pi); 1875 umtxq_unlock(&uq->uq_key); 1876 1877 /* 1878 * Care must be exercised when dealing with umtx structure. It 1879 * can fault on any access. 1880 */ 1881 for (;;) { 1882 /* 1883 * Try the uncontested case. This should be done in userland. 1884 */ 1885 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); 1886 /* The address was invalid. */ 1887 if (rv == -1) { 1888 error = EFAULT; 1889 break; 1890 } 1891 /* The acquire succeeded. */ 1892 if (rv == 0) { 1893 MPASS(owner == UMUTEX_UNOWNED); 1894 error = 0; 1895 break; 1896 } 1897 1898 if (owner == UMUTEX_RB_NOTRECOV) { 1899 error = ENOTRECOVERABLE; 1900 break; 1901 } 1902 1903 /* 1904 * Avoid overwriting a possible error from sleep due 1905 * to the pending signal with suspension check result. 1906 */ 1907 if (error == 0) { 1908 error = umtxq_check_susp(td, true); 1909 if (error != 0) 1910 break; 1911 } 1912 1913 /* If no one owns it but it is contested try to acquire it. */ 1914 if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { 1915 old_owner = owner; 1916 rv = casueword32(&m->m_owner, owner, &owner, 1917 id | UMUTEX_CONTESTED); 1918 /* The address was invalid. */ 1919 if (rv == -1) { 1920 error = EFAULT; 1921 break; 1922 } 1923 if (rv == 1) { 1924 if (error == 0) { 1925 error = umtxq_check_susp(td, true); 1926 if (error != 0) 1927 break; 1928 } 1929 1930 /* 1931 * If this failed the lock could 1932 * changed, restart. 1933 */ 1934 continue; 1935 } 1936 1937 MPASS(rv == 0); 1938 MPASS(owner == old_owner); 1939 umtxq_lock(&uq->uq_key); 1940 umtxq_busy(&uq->uq_key); 1941 error = umtx_pi_claim(pi, td); 1942 umtxq_unbusy(&uq->uq_key); 1943 umtxq_unlock(&uq->uq_key); 1944 if (error != 0) { 1945 /* 1946 * Since we're going to return an 1947 * error, restore the m_owner to its 1948 * previous, unowned state to avoid 1949 * compounding the problem. 1950 */ 1951 (void)casuword32(&m->m_owner, 1952 id | UMUTEX_CONTESTED, old_owner); 1953 } 1954 if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD) 1955 error = EOWNERDEAD; 1956 break; 1957 } 1958 1959 if ((owner & ~UMUTEX_CONTESTED) == id) { 1960 error = EDEADLK; 1961 break; 1962 } 1963 1964 if (try != 0) { 1965 error = EBUSY; 1966 break; 1967 } 1968 1969 /* 1970 * If we caught a signal, we have retried and now 1971 * exit immediately. 1972 */ 1973 if (error != 0) 1974 break; 1975 1976 umtxq_lock(&uq->uq_key); 1977 umtxq_busy(&uq->uq_key); 1978 umtxq_unlock(&uq->uq_key); 1979 1980 /* 1981 * Set the contested bit so that a release in user space 1982 * knows to use the system call for unlock. If this fails 1983 * either some one else has acquired the lock or it has been 1984 * released. 1985 */ 1986 rv = casueword32(&m->m_owner, owner, &old, owner | 1987 UMUTEX_CONTESTED); 1988 1989 /* The address was invalid. */ 1990 if (rv == -1) { 1991 umtxq_unbusy_unlocked(&uq->uq_key); 1992 error = EFAULT; 1993 break; 1994 } 1995 if (rv == 1) { 1996 umtxq_unbusy_unlocked(&uq->uq_key); 1997 error = umtxq_check_susp(td, true); 1998 if (error != 0) 1999 break; 2000 2001 /* 2002 * The lock changed and we need to retry or we 2003 * lost a race to the thread unlocking the 2004 * umtx. Note that the UMUTEX_RB_OWNERDEAD 2005 * value for owner is impossible there. 2006 */ 2007 continue; 2008 } 2009 2010 umtxq_lock(&uq->uq_key); 2011 2012 /* We set the contested bit, sleep. */ 2013 MPASS(old == owner); 2014 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, 2015 "umtxpi", timeout == NULL ? NULL : &timo, 2016 (flags & USYNC_PROCESS_SHARED) != 0); 2017 if (error != 0) 2018 continue; 2019 2020 error = umtxq_check_susp(td, false); 2021 if (error != 0) 2022 break; 2023 } 2024 2025 umtxq_lock(&uq->uq_key); 2026 umtx_pi_unref(pi); 2027 umtxq_unlock(&uq->uq_key); 2028 2029 umtx_key_release(&uq->uq_key); 2030 return (error); 2031 } 2032 2033 /* 2034 * Unlock a PI mutex. 2035 */ 2036 static int 2037 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb) 2038 { 2039 struct umtx_key key; 2040 struct umtx_q *uq_first, *uq_first2, *uq_me; 2041 struct umtx_pi *pi, *pi2; 2042 uint32_t id, new_owner, old, owner; 2043 int count, error, pri; 2044 2045 id = td->td_tid; 2046 2047 usrloop: 2048 /* 2049 * Make sure we own this mtx. 2050 */ 2051 error = fueword32(&m->m_owner, &owner); 2052 if (error == -1) 2053 return (EFAULT); 2054 2055 if ((owner & ~UMUTEX_CONTESTED) != id) 2056 return (EPERM); 2057 2058 new_owner = umtx_unlock_val(flags, rb); 2059 2060 /* This should be done in userland */ 2061 if ((owner & UMUTEX_CONTESTED) == 0) { 2062 error = casueword32(&m->m_owner, owner, &old, new_owner); 2063 if (error == -1) 2064 return (EFAULT); 2065 if (error == 1) { 2066 error = umtxq_check_susp(td, true); 2067 if (error != 0) 2068 return (error); 2069 goto usrloop; 2070 } 2071 if (old == owner) 2072 return (0); 2073 owner = old; 2074 } 2075 2076 /* We should only ever be in here for contested locks */ 2077 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2078 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), 2079 &key)) != 0) 2080 return (error); 2081 2082 umtxq_lock(&key); 2083 umtxq_busy(&key); 2084 count = umtxq_count_pi(&key, &uq_first); 2085 if (uq_first != NULL) { 2086 mtx_lock(&umtx_lock); 2087 pi = uq_first->uq_pi_blocked; 2088 KASSERT(pi != NULL, ("pi == NULL?")); 2089 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { 2090 mtx_unlock(&umtx_lock); 2091 umtxq_unbusy(&key); 2092 umtxq_unlock(&key); 2093 umtx_key_release(&key); 2094 /* userland messed the mutex */ 2095 return (EPERM); 2096 } 2097 uq_me = td->td_umtxq; 2098 if (pi->pi_owner == td) 2099 umtx_pi_disown(pi); 2100 /* get highest priority thread which is still sleeping. */ 2101 uq_first = TAILQ_FIRST(&pi->pi_blocked); 2102 while (uq_first != NULL && 2103 (uq_first->uq_flags & UQF_UMTXQ) == 0) { 2104 uq_first = TAILQ_NEXT(uq_first, uq_lockq); 2105 } 2106 pri = PRI_MAX; 2107 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { 2108 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); 2109 if (uq_first2 != NULL) { 2110 if (pri > UPRI(uq_first2->uq_thread)) 2111 pri = UPRI(uq_first2->uq_thread); 2112 } 2113 } 2114 thread_lock(td); 2115 sched_lend_user_prio(td, pri); 2116 thread_unlock(td); 2117 mtx_unlock(&umtx_lock); 2118 if (uq_first) 2119 umtxq_signal_thread(uq_first); 2120 } else { 2121 pi = umtx_pi_lookup(&key); 2122 /* 2123 * A umtx_pi can exist if a signal or timeout removed the 2124 * last waiter from the umtxq, but there is still 2125 * a thread in do_lock_pi() holding the umtx_pi. 2126 */ 2127 if (pi != NULL) { 2128 /* 2129 * The umtx_pi can be unowned, such as when a thread 2130 * has just entered do_lock_pi(), allocated the 2131 * umtx_pi, and unlocked the umtxq. 2132 * If the current thread owns it, it must disown it. 2133 */ 2134 mtx_lock(&umtx_lock); 2135 if (pi->pi_owner == td) 2136 umtx_pi_disown(pi); 2137 mtx_unlock(&umtx_lock); 2138 } 2139 } 2140 umtxq_unlock(&key); 2141 2142 /* 2143 * When unlocking the umtx, it must be marked as unowned if 2144 * there is zero or one thread only waiting for it. 2145 * Otherwise, it must be marked as contested. 2146 */ 2147 2148 if (count > 1) 2149 new_owner |= UMUTEX_CONTESTED; 2150 again: 2151 error = casueword32(&m->m_owner, owner, &old, new_owner); 2152 if (error == 1) { 2153 error = umtxq_check_susp(td, false); 2154 if (error == 0) 2155 goto again; 2156 } 2157 umtxq_unbusy_unlocked(&key); 2158 umtx_key_release(&key); 2159 if (error == -1) 2160 return (EFAULT); 2161 if (error == 0 && old != owner) 2162 return (EINVAL); 2163 return (error); 2164 } 2165 2166 /* 2167 * Lock a PP mutex. 2168 */ 2169 static int 2170 do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, 2171 struct _umtx_time *timeout, int try) 2172 { 2173 struct abs_timeout timo; 2174 struct umtx_q *uq, *uq2; 2175 struct umtx_pi *pi; 2176 uint32_t ceiling; 2177 uint32_t owner, id; 2178 int error, pri, old_inherited_pri, su, rv; 2179 2180 id = td->td_tid; 2181 uq = td->td_umtxq; 2182 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2183 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), 2184 &uq->uq_key)) != 0) 2185 return (error); 2186 2187 if (timeout != NULL) 2188 abs_timeout_init2(&timo, timeout); 2189 2190 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 2191 for (;;) { 2192 old_inherited_pri = uq->uq_inherited_pri; 2193 umtxq_lock(&uq->uq_key); 2194 umtxq_busy(&uq->uq_key); 2195 umtxq_unlock(&uq->uq_key); 2196 2197 rv = fueword32(&m->m_ceilings[0], &ceiling); 2198 if (rv == -1) { 2199 error = EFAULT; 2200 goto out; 2201 } 2202 ceiling = RTP_PRIO_MAX - ceiling; 2203 if (ceiling > RTP_PRIO_MAX) { 2204 error = EINVAL; 2205 goto out; 2206 } 2207 2208 mtx_lock(&umtx_lock); 2209 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { 2210 mtx_unlock(&umtx_lock); 2211 error = EINVAL; 2212 goto out; 2213 } 2214 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { 2215 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; 2216 thread_lock(td); 2217 if (uq->uq_inherited_pri < UPRI(td)) 2218 sched_lend_user_prio(td, uq->uq_inherited_pri); 2219 thread_unlock(td); 2220 } 2221 mtx_unlock(&umtx_lock); 2222 2223 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, 2224 id | UMUTEX_CONTESTED); 2225 /* The address was invalid. */ 2226 if (rv == -1) { 2227 error = EFAULT; 2228 break; 2229 } 2230 if (rv == 0) { 2231 MPASS(owner == UMUTEX_CONTESTED); 2232 error = 0; 2233 break; 2234 } 2235 /* rv == 1 */ 2236 if (owner == UMUTEX_RB_OWNERDEAD) { 2237 rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD, 2238 &owner, id | UMUTEX_CONTESTED); 2239 if (rv == -1) { 2240 error = EFAULT; 2241 break; 2242 } 2243 if (rv == 0) { 2244 MPASS(owner == UMUTEX_RB_OWNERDEAD); 2245 error = EOWNERDEAD; /* success */ 2246 break; 2247 } 2248 2249 /* 2250 * rv == 1, only check for suspension if we 2251 * did not already catched a signal. If we 2252 * get an error from the check, the same 2253 * condition is checked by the umtxq_sleep() 2254 * call below, so we should obliterate the 2255 * error to not skip the last loop iteration. 2256 */ 2257 if (error == 0) { 2258 error = umtxq_check_susp(td, false); 2259 if (error == 0) { 2260 if (try != 0) 2261 error = EBUSY; 2262 else 2263 continue; 2264 } 2265 error = 0; 2266 } 2267 } else if (owner == UMUTEX_RB_NOTRECOV) { 2268 error = ENOTRECOVERABLE; 2269 } 2270 2271 if (try != 0) 2272 error = EBUSY; 2273 2274 /* 2275 * If we caught a signal, we have retried and now 2276 * exit immediately. 2277 */ 2278 if (error != 0) 2279 break; 2280 2281 umtxq_lock(&uq->uq_key); 2282 umtxq_insert(uq); 2283 umtxq_unbusy(&uq->uq_key); 2284 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ? 2285 NULL : &timo); 2286 umtxq_remove(uq); 2287 umtxq_unlock(&uq->uq_key); 2288 2289 mtx_lock(&umtx_lock); 2290 uq->uq_inherited_pri = old_inherited_pri; 2291 pri = PRI_MAX; 2292 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2293 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2294 if (uq2 != NULL) { 2295 if (pri > UPRI(uq2->uq_thread)) 2296 pri = UPRI(uq2->uq_thread); 2297 } 2298 } 2299 if (pri > uq->uq_inherited_pri) 2300 pri = uq->uq_inherited_pri; 2301 thread_lock(td); 2302 sched_lend_user_prio(td, pri); 2303 thread_unlock(td); 2304 mtx_unlock(&umtx_lock); 2305 } 2306 2307 if (error != 0 && error != EOWNERDEAD) { 2308 mtx_lock(&umtx_lock); 2309 uq->uq_inherited_pri = old_inherited_pri; 2310 pri = PRI_MAX; 2311 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2312 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2313 if (uq2 != NULL) { 2314 if (pri > UPRI(uq2->uq_thread)) 2315 pri = UPRI(uq2->uq_thread); 2316 } 2317 } 2318 if (pri > uq->uq_inherited_pri) 2319 pri = uq->uq_inherited_pri; 2320 thread_lock(td); 2321 sched_lend_user_prio(td, pri); 2322 thread_unlock(td); 2323 mtx_unlock(&umtx_lock); 2324 } 2325 2326 out: 2327 umtxq_unbusy_unlocked(&uq->uq_key); 2328 umtx_key_release(&uq->uq_key); 2329 return (error); 2330 } 2331 2332 /* 2333 * Unlock a PP mutex. 2334 */ 2335 static int 2336 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb) 2337 { 2338 struct umtx_key key; 2339 struct umtx_q *uq, *uq2; 2340 struct umtx_pi *pi; 2341 uint32_t id, owner, rceiling; 2342 int error, pri, new_inherited_pri, su; 2343 2344 id = td->td_tid; 2345 uq = td->td_umtxq; 2346 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 2347 2348 /* 2349 * Make sure we own this mtx. 2350 */ 2351 error = fueword32(&m->m_owner, &owner); 2352 if (error == -1) 2353 return (EFAULT); 2354 2355 if ((owner & ~UMUTEX_CONTESTED) != id) 2356 return (EPERM); 2357 2358 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); 2359 if (error != 0) 2360 return (error); 2361 2362 if (rceiling == -1) 2363 new_inherited_pri = PRI_MAX; 2364 else { 2365 rceiling = RTP_PRIO_MAX - rceiling; 2366 if (rceiling > RTP_PRIO_MAX) 2367 return (EINVAL); 2368 new_inherited_pri = PRI_MIN_REALTIME + rceiling; 2369 } 2370 2371 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2372 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), 2373 &key)) != 0) 2374 return (error); 2375 umtxq_lock(&key); 2376 umtxq_busy(&key); 2377 umtxq_unlock(&key); 2378 /* 2379 * For priority protected mutex, always set unlocked state 2380 * to UMUTEX_CONTESTED, so that userland always enters kernel 2381 * to lock the mutex, it is necessary because thread priority 2382 * has to be adjusted for such mutex. 2383 */ 2384 error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) | 2385 UMUTEX_CONTESTED); 2386 2387 umtxq_lock(&key); 2388 if (error == 0) 2389 umtxq_signal(&key, 1); 2390 umtxq_unbusy(&key); 2391 umtxq_unlock(&key); 2392 2393 if (error == -1) 2394 error = EFAULT; 2395 else { 2396 mtx_lock(&umtx_lock); 2397 if (su != 0) 2398 uq->uq_inherited_pri = new_inherited_pri; 2399 pri = PRI_MAX; 2400 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2401 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2402 if (uq2 != NULL) { 2403 if (pri > UPRI(uq2->uq_thread)) 2404 pri = UPRI(uq2->uq_thread); 2405 } 2406 } 2407 if (pri > uq->uq_inherited_pri) 2408 pri = uq->uq_inherited_pri; 2409 thread_lock(td); 2410 sched_lend_user_prio(td, pri); 2411 thread_unlock(td); 2412 mtx_unlock(&umtx_lock); 2413 } 2414 umtx_key_release(&key); 2415 return (error); 2416 } 2417 2418 static int 2419 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, 2420 uint32_t *old_ceiling) 2421 { 2422 struct umtx_q *uq; 2423 uint32_t flags, id, owner, save_ceiling; 2424 int error, rv, rv1; 2425 2426 error = fueword32(&m->m_flags, &flags); 2427 if (error == -1) 2428 return (EFAULT); 2429 if ((flags & UMUTEX_PRIO_PROTECT) == 0) 2430 return (EINVAL); 2431 if (ceiling > RTP_PRIO_MAX) 2432 return (EINVAL); 2433 id = td->td_tid; 2434 uq = td->td_umtxq; 2435 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2436 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), 2437 &uq->uq_key)) != 0) 2438 return (error); 2439 for (;;) { 2440 umtxq_lock(&uq->uq_key); 2441 umtxq_busy(&uq->uq_key); 2442 umtxq_unlock(&uq->uq_key); 2443 2444 rv = fueword32(&m->m_ceilings[0], &save_ceiling); 2445 if (rv == -1) { 2446 error = EFAULT; 2447 break; 2448 } 2449 2450 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, 2451 id | UMUTEX_CONTESTED); 2452 if (rv == -1) { 2453 error = EFAULT; 2454 break; 2455 } 2456 2457 if (rv == 0) { 2458 MPASS(owner == UMUTEX_CONTESTED); 2459 rv = suword32(&m->m_ceilings[0], ceiling); 2460 rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED); 2461 error = (rv == 0 && rv1 == 0) ? 0: EFAULT; 2462 break; 2463 } 2464 2465 if ((owner & ~UMUTEX_CONTESTED) == id) { 2466 rv = suword32(&m->m_ceilings[0], ceiling); 2467 error = rv == 0 ? 0 : EFAULT; 2468 break; 2469 } 2470 2471 if (owner == UMUTEX_RB_OWNERDEAD) { 2472 error = EOWNERDEAD; 2473 break; 2474 } else if (owner == UMUTEX_RB_NOTRECOV) { 2475 error = ENOTRECOVERABLE; 2476 break; 2477 } 2478 2479 /* 2480 * If we caught a signal, we have retried and now 2481 * exit immediately. 2482 */ 2483 if (error != 0) 2484 break; 2485 2486 /* 2487 * We set the contested bit, sleep. Otherwise the lock changed 2488 * and we need to retry or we lost a race to the thread 2489 * unlocking the umtx. 2490 */ 2491 umtxq_lock(&uq->uq_key); 2492 umtxq_insert(uq); 2493 umtxq_unbusy(&uq->uq_key); 2494 error = umtxq_sleep(uq, "umtxpp", NULL); 2495 umtxq_remove(uq); 2496 umtxq_unlock(&uq->uq_key); 2497 } 2498 umtxq_lock(&uq->uq_key); 2499 if (error == 0) 2500 umtxq_signal(&uq->uq_key, INT_MAX); 2501 umtxq_unbusy(&uq->uq_key); 2502 umtxq_unlock(&uq->uq_key); 2503 umtx_key_release(&uq->uq_key); 2504 if (error == 0 && old_ceiling != NULL) { 2505 rv = suword32(old_ceiling, save_ceiling); 2506 error = rv == 0 ? 0 : EFAULT; 2507 } 2508 return (error); 2509 } 2510 2511 /* 2512 * Lock a userland POSIX mutex. 2513 */ 2514 static int 2515 do_lock_umutex(struct thread *td, struct umutex *m, 2516 struct _umtx_time *timeout, int mode) 2517 { 2518 uint32_t flags; 2519 int error; 2520 2521 error = fueword32(&m->m_flags, &flags); 2522 if (error == -1) 2523 return (EFAULT); 2524 2525 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2526 case 0: 2527 error = do_lock_normal(td, m, flags, timeout, mode); 2528 break; 2529 case UMUTEX_PRIO_INHERIT: 2530 error = do_lock_pi(td, m, flags, timeout, mode); 2531 break; 2532 case UMUTEX_PRIO_PROTECT: 2533 error = do_lock_pp(td, m, flags, timeout, mode); 2534 break; 2535 default: 2536 return (EINVAL); 2537 } 2538 if (timeout == NULL) { 2539 if (error == EINTR && mode != _UMUTEX_WAIT) 2540 error = ERESTART; 2541 } else { 2542 /* Timed-locking is not restarted. */ 2543 if (error == ERESTART) 2544 error = EINTR; 2545 } 2546 return (error); 2547 } 2548 2549 /* 2550 * Unlock a userland POSIX mutex. 2551 */ 2552 static int 2553 do_unlock_umutex(struct thread *td, struct umutex *m, bool rb) 2554 { 2555 uint32_t flags; 2556 int error; 2557 2558 error = fueword32(&m->m_flags, &flags); 2559 if (error == -1) 2560 return (EFAULT); 2561 2562 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2563 case 0: 2564 return (do_unlock_normal(td, m, flags, rb)); 2565 case UMUTEX_PRIO_INHERIT: 2566 return (do_unlock_pi(td, m, flags, rb)); 2567 case UMUTEX_PRIO_PROTECT: 2568 return (do_unlock_pp(td, m, flags, rb)); 2569 } 2570 2571 return (EINVAL); 2572 } 2573 2574 static int 2575 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, 2576 struct timespec *timeout, u_long wflags) 2577 { 2578 struct abs_timeout timo; 2579 struct umtx_q *uq; 2580 uint32_t flags, clockid, hasw; 2581 int error; 2582 2583 uq = td->td_umtxq; 2584 error = fueword32(&cv->c_flags, &flags); 2585 if (error == -1) 2586 return (EFAULT); 2587 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); 2588 if (error != 0) 2589 return (error); 2590 2591 if ((wflags & CVWAIT_CLOCKID) != 0) { 2592 error = fueword32(&cv->c_clockid, &clockid); 2593 if (error == -1) { 2594 umtx_key_release(&uq->uq_key); 2595 return (EFAULT); 2596 } 2597 if (clockid < CLOCK_REALTIME || 2598 clockid >= CLOCK_THREAD_CPUTIME_ID) { 2599 /* hmm, only HW clock id will work. */ 2600 umtx_key_release(&uq->uq_key); 2601 return (EINVAL); 2602 } 2603 } else { 2604 clockid = CLOCK_REALTIME; 2605 } 2606 2607 umtxq_lock(&uq->uq_key); 2608 umtxq_busy(&uq->uq_key); 2609 umtxq_insert(uq); 2610 umtxq_unlock(&uq->uq_key); 2611 2612 /* 2613 * Set c_has_waiters to 1 before releasing user mutex, also 2614 * don't modify cache line when unnecessary. 2615 */ 2616 error = fueword32(&cv->c_has_waiters, &hasw); 2617 if (error == 0 && hasw == 0) 2618 suword32(&cv->c_has_waiters, 1); 2619 2620 umtxq_unbusy_unlocked(&uq->uq_key); 2621 2622 error = do_unlock_umutex(td, m, false); 2623 2624 if (timeout != NULL) 2625 abs_timeout_init(&timo, clockid, (wflags & CVWAIT_ABSTIME) != 0, 2626 timeout); 2627 2628 umtxq_lock(&uq->uq_key); 2629 if (error == 0) { 2630 error = umtxq_sleep(uq, "ucond", timeout == NULL ? 2631 NULL : &timo); 2632 } 2633 2634 if ((uq->uq_flags & UQF_UMTXQ) == 0) 2635 error = 0; 2636 else { 2637 /* 2638 * This must be timeout,interrupted by signal or 2639 * surprious wakeup, clear c_has_waiter flag when 2640 * necessary. 2641 */ 2642 umtxq_busy(&uq->uq_key); 2643 if ((uq->uq_flags & UQF_UMTXQ) != 0) { 2644 int oldlen = uq->uq_cur_queue->length; 2645 umtxq_remove(uq); 2646 if (oldlen == 1) { 2647 umtxq_unlock(&uq->uq_key); 2648 suword32(&cv->c_has_waiters, 0); 2649 umtxq_lock(&uq->uq_key); 2650 } 2651 } 2652 umtxq_unbusy(&uq->uq_key); 2653 if (error == ERESTART) 2654 error = EINTR; 2655 } 2656 2657 umtxq_unlock(&uq->uq_key); 2658 umtx_key_release(&uq->uq_key); 2659 return (error); 2660 } 2661 2662 /* 2663 * Signal a userland condition variable. 2664 */ 2665 static int 2666 do_cv_signal(struct thread *td, struct ucond *cv) 2667 { 2668 struct umtx_key key; 2669 int error, cnt, nwake; 2670 uint32_t flags; 2671 2672 error = fueword32(&cv->c_flags, &flags); 2673 if (error == -1) 2674 return (EFAULT); 2675 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2676 return (error); 2677 umtxq_lock(&key); 2678 umtxq_busy(&key); 2679 cnt = umtxq_count(&key); 2680 nwake = umtxq_signal(&key, 1); 2681 if (cnt <= nwake) { 2682 umtxq_unlock(&key); 2683 error = suword32(&cv->c_has_waiters, 0); 2684 if (error == -1) 2685 error = EFAULT; 2686 umtxq_lock(&key); 2687 } 2688 umtxq_unbusy(&key); 2689 umtxq_unlock(&key); 2690 umtx_key_release(&key); 2691 return (error); 2692 } 2693 2694 static int 2695 do_cv_broadcast(struct thread *td, struct ucond *cv) 2696 { 2697 struct umtx_key key; 2698 int error; 2699 uint32_t flags; 2700 2701 error = fueword32(&cv->c_flags, &flags); 2702 if (error == -1) 2703 return (EFAULT); 2704 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2705 return (error); 2706 2707 umtxq_lock(&key); 2708 umtxq_busy(&key); 2709 umtxq_signal(&key, INT_MAX); 2710 umtxq_unlock(&key); 2711 2712 error = suword32(&cv->c_has_waiters, 0); 2713 if (error == -1) 2714 error = EFAULT; 2715 2716 umtxq_unbusy_unlocked(&key); 2717 2718 umtx_key_release(&key); 2719 return (error); 2720 } 2721 2722 static int 2723 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, 2724 struct _umtx_time *timeout) 2725 { 2726 struct abs_timeout timo; 2727 struct umtx_q *uq; 2728 uint32_t flags, wrflags; 2729 int32_t state, oldstate; 2730 int32_t blocked_readers; 2731 int error, error1, rv; 2732 2733 uq = td->td_umtxq; 2734 error = fueword32(&rwlock->rw_flags, &flags); 2735 if (error == -1) 2736 return (EFAULT); 2737 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2738 if (error != 0) 2739 return (error); 2740 2741 if (timeout != NULL) 2742 abs_timeout_init2(&timo, timeout); 2743 2744 wrflags = URWLOCK_WRITE_OWNER; 2745 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER)) 2746 wrflags |= URWLOCK_WRITE_WAITERS; 2747 2748 for (;;) { 2749 rv = fueword32(&rwlock->rw_state, &state); 2750 if (rv == -1) { 2751 umtx_key_release(&uq->uq_key); 2752 return (EFAULT); 2753 } 2754 2755 /* try to lock it */ 2756 while (!(state & wrflags)) { 2757 if (__predict_false(URWLOCK_READER_COUNT(state) == 2758 URWLOCK_MAX_READERS)) { 2759 umtx_key_release(&uq->uq_key); 2760 return (EAGAIN); 2761 } 2762 rv = casueword32(&rwlock->rw_state, state, 2763 &oldstate, state + 1); 2764 if (rv == -1) { 2765 umtx_key_release(&uq->uq_key); 2766 return (EFAULT); 2767 } 2768 if (rv == 0) { 2769 MPASS(oldstate == state); 2770 umtx_key_release(&uq->uq_key); 2771 return (0); 2772 } 2773 error = umtxq_check_susp(td, true); 2774 if (error != 0) 2775 break; 2776 state = oldstate; 2777 } 2778 2779 if (error) 2780 break; 2781 2782 /* grab monitor lock */ 2783 umtxq_lock(&uq->uq_key); 2784 umtxq_busy(&uq->uq_key); 2785 umtxq_unlock(&uq->uq_key); 2786 2787 /* 2788 * re-read the state, in case it changed between the try-lock above 2789 * and the check below 2790 */ 2791 rv = fueword32(&rwlock->rw_state, &state); 2792 if (rv == -1) 2793 error = EFAULT; 2794 2795 /* set read contention bit */ 2796 while (error == 0 && (state & wrflags) && 2797 !(state & URWLOCK_READ_WAITERS)) { 2798 rv = casueword32(&rwlock->rw_state, state, 2799 &oldstate, state | URWLOCK_READ_WAITERS); 2800 if (rv == -1) { 2801 error = EFAULT; 2802 break; 2803 } 2804 if (rv == 0) { 2805 MPASS(oldstate == state); 2806 goto sleep; 2807 } 2808 state = oldstate; 2809 error = umtxq_check_susp(td, false); 2810 if (error != 0) 2811 break; 2812 } 2813 if (error != 0) { 2814 umtxq_unbusy_unlocked(&uq->uq_key); 2815 break; 2816 } 2817 2818 /* state is changed while setting flags, restart */ 2819 if (!(state & wrflags)) { 2820 umtxq_unbusy_unlocked(&uq->uq_key); 2821 error = umtxq_check_susp(td, true); 2822 if (error != 0) 2823 break; 2824 continue; 2825 } 2826 2827 sleep: 2828 /* 2829 * Contention bit is set, before sleeping, increase 2830 * read waiter count. 2831 */ 2832 rv = fueword32(&rwlock->rw_blocked_readers, 2833 &blocked_readers); 2834 if (rv == -1) { 2835 umtxq_unbusy_unlocked(&uq->uq_key); 2836 error = EFAULT; 2837 break; 2838 } 2839 suword32(&rwlock->rw_blocked_readers, blocked_readers+1); 2840 2841 while (state & wrflags) { 2842 umtxq_lock(&uq->uq_key); 2843 umtxq_insert(uq); 2844 umtxq_unbusy(&uq->uq_key); 2845 2846 error = umtxq_sleep(uq, "urdlck", timeout == NULL ? 2847 NULL : &timo); 2848 2849 umtxq_busy(&uq->uq_key); 2850 umtxq_remove(uq); 2851 umtxq_unlock(&uq->uq_key); 2852 if (error) 2853 break; 2854 rv = fueword32(&rwlock->rw_state, &state); 2855 if (rv == -1) { 2856 error = EFAULT; 2857 break; 2858 } 2859 } 2860 2861 /* decrease read waiter count, and may clear read contention bit */ 2862 rv = fueword32(&rwlock->rw_blocked_readers, 2863 &blocked_readers); 2864 if (rv == -1) { 2865 umtxq_unbusy_unlocked(&uq->uq_key); 2866 error = EFAULT; 2867 break; 2868 } 2869 suword32(&rwlock->rw_blocked_readers, blocked_readers-1); 2870 if (blocked_readers == 1) { 2871 rv = fueword32(&rwlock->rw_state, &state); 2872 if (rv == -1) { 2873 umtxq_unbusy_unlocked(&uq->uq_key); 2874 error = EFAULT; 2875 break; 2876 } 2877 for (;;) { 2878 rv = casueword32(&rwlock->rw_state, state, 2879 &oldstate, state & ~URWLOCK_READ_WAITERS); 2880 if (rv == -1) { 2881 error = EFAULT; 2882 break; 2883 } 2884 if (rv == 0) { 2885 MPASS(oldstate == state); 2886 break; 2887 } 2888 state = oldstate; 2889 error1 = umtxq_check_susp(td, false); 2890 if (error1 != 0) { 2891 if (error == 0) 2892 error = error1; 2893 break; 2894 } 2895 } 2896 } 2897 2898 umtxq_unbusy_unlocked(&uq->uq_key); 2899 if (error != 0) 2900 break; 2901 } 2902 umtx_key_release(&uq->uq_key); 2903 if (error == ERESTART) 2904 error = EINTR; 2905 return (error); 2906 } 2907 2908 static int 2909 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout) 2910 { 2911 struct abs_timeout timo; 2912 struct umtx_q *uq; 2913 uint32_t flags; 2914 int32_t state, oldstate; 2915 int32_t blocked_writers; 2916 int32_t blocked_readers; 2917 int error, error1, rv; 2918 2919 uq = td->td_umtxq; 2920 error = fueword32(&rwlock->rw_flags, &flags); 2921 if (error == -1) 2922 return (EFAULT); 2923 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2924 if (error != 0) 2925 return (error); 2926 2927 if (timeout != NULL) 2928 abs_timeout_init2(&timo, timeout); 2929 2930 blocked_readers = 0; 2931 for (;;) { 2932 rv = fueword32(&rwlock->rw_state, &state); 2933 if (rv == -1) { 2934 umtx_key_release(&uq->uq_key); 2935 return (EFAULT); 2936 } 2937 while ((state & URWLOCK_WRITE_OWNER) == 0 && 2938 URWLOCK_READER_COUNT(state) == 0) { 2939 rv = casueword32(&rwlock->rw_state, state, 2940 &oldstate, state | URWLOCK_WRITE_OWNER); 2941 if (rv == -1) { 2942 umtx_key_release(&uq->uq_key); 2943 return (EFAULT); 2944 } 2945 if (rv == 0) { 2946 MPASS(oldstate == state); 2947 umtx_key_release(&uq->uq_key); 2948 return (0); 2949 } 2950 state = oldstate; 2951 error = umtxq_check_susp(td, true); 2952 if (error != 0) 2953 break; 2954 } 2955 2956 if (error) { 2957 if ((state & (URWLOCK_WRITE_OWNER | 2958 URWLOCK_WRITE_WAITERS)) == 0 && 2959 blocked_readers != 0) { 2960 umtxq_lock(&uq->uq_key); 2961 umtxq_busy(&uq->uq_key); 2962 umtxq_signal_queue(&uq->uq_key, INT_MAX, 2963 UMTX_SHARED_QUEUE); 2964 umtxq_unbusy(&uq->uq_key); 2965 umtxq_unlock(&uq->uq_key); 2966 } 2967 2968 break; 2969 } 2970 2971 /* grab monitor lock */ 2972 umtxq_lock(&uq->uq_key); 2973 umtxq_busy(&uq->uq_key); 2974 umtxq_unlock(&uq->uq_key); 2975 2976 /* 2977 * Re-read the state, in case it changed between the 2978 * try-lock above and the check below. 2979 */ 2980 rv = fueword32(&rwlock->rw_state, &state); 2981 if (rv == -1) 2982 error = EFAULT; 2983 2984 while (error == 0 && ((state & URWLOCK_WRITE_OWNER) || 2985 URWLOCK_READER_COUNT(state) != 0) && 2986 (state & URWLOCK_WRITE_WAITERS) == 0) { 2987 rv = casueword32(&rwlock->rw_state, state, 2988 &oldstate, state | URWLOCK_WRITE_WAITERS); 2989 if (rv == -1) { 2990 error = EFAULT; 2991 break; 2992 } 2993 if (rv == 0) { 2994 MPASS(oldstate == state); 2995 goto sleep; 2996 } 2997 state = oldstate; 2998 error = umtxq_check_susp(td, false); 2999 if (error != 0) 3000 break; 3001 } 3002 if (error != 0) { 3003 umtxq_unbusy_unlocked(&uq->uq_key); 3004 break; 3005 } 3006 3007 if ((state & URWLOCK_WRITE_OWNER) == 0 && 3008 URWLOCK_READER_COUNT(state) == 0) { 3009 umtxq_unbusy_unlocked(&uq->uq_key); 3010 error = umtxq_check_susp(td, false); 3011 if (error != 0) 3012 break; 3013 continue; 3014 } 3015 sleep: 3016 rv = fueword32(&rwlock->rw_blocked_writers, 3017 &blocked_writers); 3018 if (rv == -1) { 3019 umtxq_unbusy_unlocked(&uq->uq_key); 3020 error = EFAULT; 3021 break; 3022 } 3023 suword32(&rwlock->rw_blocked_writers, blocked_writers + 1); 3024 3025 while ((state & URWLOCK_WRITE_OWNER) || 3026 URWLOCK_READER_COUNT(state) != 0) { 3027 umtxq_lock(&uq->uq_key); 3028 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE); 3029 umtxq_unbusy(&uq->uq_key); 3030 3031 error = umtxq_sleep(uq, "uwrlck", timeout == NULL ? 3032 NULL : &timo); 3033 3034 umtxq_busy(&uq->uq_key); 3035 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE); 3036 umtxq_unlock(&uq->uq_key); 3037 if (error) 3038 break; 3039 rv = fueword32(&rwlock->rw_state, &state); 3040 if (rv == -1) { 3041 error = EFAULT; 3042 break; 3043 } 3044 } 3045 3046 rv = fueword32(&rwlock->rw_blocked_writers, 3047 &blocked_writers); 3048 if (rv == -1) { 3049 umtxq_unbusy_unlocked(&uq->uq_key); 3050 error = EFAULT; 3051 break; 3052 } 3053 suword32(&rwlock->rw_blocked_writers, blocked_writers-1); 3054 if (blocked_writers == 1) { 3055 rv = fueword32(&rwlock->rw_state, &state); 3056 if (rv == -1) { 3057 umtxq_unbusy_unlocked(&uq->uq_key); 3058 error = EFAULT; 3059 break; 3060 } 3061 for (;;) { 3062 rv = casueword32(&rwlock->rw_state, state, 3063 &oldstate, state & ~URWLOCK_WRITE_WAITERS); 3064 if (rv == -1) { 3065 error = EFAULT; 3066 break; 3067 } 3068 if (rv == 0) { 3069 MPASS(oldstate == state); 3070 break; 3071 } 3072 state = oldstate; 3073 error1 = umtxq_check_susp(td, false); 3074 /* 3075 * We are leaving the URWLOCK_WRITE_WAITERS 3076 * behind, but this should not harm the 3077 * correctness. 3078 */ 3079 if (error1 != 0) { 3080 if (error == 0) 3081 error = error1; 3082 break; 3083 } 3084 } 3085 rv = fueword32(&rwlock->rw_blocked_readers, 3086 &blocked_readers); 3087 if (rv == -1) { 3088 umtxq_unbusy_unlocked(&uq->uq_key); 3089 error = EFAULT; 3090 break; 3091 } 3092 } else 3093 blocked_readers = 0; 3094 3095 umtxq_unbusy_unlocked(&uq->uq_key); 3096 } 3097 3098 umtx_key_release(&uq->uq_key); 3099 if (error == ERESTART) 3100 error = EINTR; 3101 return (error); 3102 } 3103 3104 static int 3105 do_rw_unlock(struct thread *td, struct urwlock *rwlock) 3106 { 3107 struct umtx_q *uq; 3108 uint32_t flags; 3109 int32_t state, oldstate; 3110 int error, rv, q, count; 3111 3112 uq = td->td_umtxq; 3113 error = fueword32(&rwlock->rw_flags, &flags); 3114 if (error == -1) 3115 return (EFAULT); 3116 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 3117 if (error != 0) 3118 return (error); 3119 3120 error = fueword32(&rwlock->rw_state, &state); 3121 if (error == -1) { 3122 error = EFAULT; 3123 goto out; 3124 } 3125 if (state & URWLOCK_WRITE_OWNER) { 3126 for (;;) { 3127 rv = casueword32(&rwlock->rw_state, state, 3128 &oldstate, state & ~URWLOCK_WRITE_OWNER); 3129 if (rv == -1) { 3130 error = EFAULT; 3131 goto out; 3132 } 3133 if (rv == 1) { 3134 state = oldstate; 3135 if (!(oldstate & URWLOCK_WRITE_OWNER)) { 3136 error = EPERM; 3137 goto out; 3138 } 3139 error = umtxq_check_susp(td, true); 3140 if (error != 0) 3141 goto out; 3142 } else 3143 break; 3144 } 3145 } else if (URWLOCK_READER_COUNT(state) != 0) { 3146 for (;;) { 3147 rv = casueword32(&rwlock->rw_state, state, 3148 &oldstate, state - 1); 3149 if (rv == -1) { 3150 error = EFAULT; 3151 goto out; 3152 } 3153 if (rv == 1) { 3154 state = oldstate; 3155 if (URWLOCK_READER_COUNT(oldstate) == 0) { 3156 error = EPERM; 3157 goto out; 3158 } 3159 error = umtxq_check_susp(td, true); 3160 if (error != 0) 3161 goto out; 3162 } else 3163 break; 3164 } 3165 } else { 3166 error = EPERM; 3167 goto out; 3168 } 3169 3170 count = 0; 3171 3172 if (!(flags & URWLOCK_PREFER_READER)) { 3173 if (state & URWLOCK_WRITE_WAITERS) { 3174 count = 1; 3175 q = UMTX_EXCLUSIVE_QUEUE; 3176 } else if (state & URWLOCK_READ_WAITERS) { 3177 count = INT_MAX; 3178 q = UMTX_SHARED_QUEUE; 3179 } 3180 } else { 3181 if (state & URWLOCK_READ_WAITERS) { 3182 count = INT_MAX; 3183 q = UMTX_SHARED_QUEUE; 3184 } else if (state & URWLOCK_WRITE_WAITERS) { 3185 count = 1; 3186 q = UMTX_EXCLUSIVE_QUEUE; 3187 } 3188 } 3189 3190 if (count) { 3191 umtxq_lock(&uq->uq_key); 3192 umtxq_busy(&uq->uq_key); 3193 umtxq_signal_queue(&uq->uq_key, count, q); 3194 umtxq_unbusy(&uq->uq_key); 3195 umtxq_unlock(&uq->uq_key); 3196 } 3197 out: 3198 umtx_key_release(&uq->uq_key); 3199 return (error); 3200 } 3201 3202 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 3203 static int 3204 do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) 3205 { 3206 struct abs_timeout timo; 3207 struct umtx_q *uq; 3208 uint32_t flags, count, count1; 3209 int error, rv, rv1; 3210 3211 uq = td->td_umtxq; 3212 error = fueword32(&sem->_flags, &flags); 3213 if (error == -1) 3214 return (EFAULT); 3215 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); 3216 if (error != 0) 3217 return (error); 3218 3219 if (timeout != NULL) 3220 abs_timeout_init2(&timo, timeout); 3221 3222 again: 3223 umtxq_lock(&uq->uq_key); 3224 umtxq_busy(&uq->uq_key); 3225 umtxq_insert(uq); 3226 umtxq_unlock(&uq->uq_key); 3227 rv = casueword32(&sem->_has_waiters, 0, &count1, 1); 3228 if (rv == 0) 3229 rv1 = fueword32(&sem->_count, &count); 3230 if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || 3231 (rv == 1 && count1 == 0)) { 3232 umtxq_lock(&uq->uq_key); 3233 umtxq_unbusy(&uq->uq_key); 3234 umtxq_remove(uq); 3235 umtxq_unlock(&uq->uq_key); 3236 if (rv == 1) { 3237 rv = umtxq_check_susp(td, true); 3238 if (rv == 0) 3239 goto again; 3240 error = rv; 3241 goto out; 3242 } 3243 if (rv == 0) 3244 rv = rv1; 3245 error = rv == -1 ? EFAULT : 0; 3246 goto out; 3247 } 3248 umtxq_lock(&uq->uq_key); 3249 umtxq_unbusy(&uq->uq_key); 3250 3251 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); 3252 3253 if ((uq->uq_flags & UQF_UMTXQ) == 0) 3254 error = 0; 3255 else { 3256 umtxq_remove(uq); 3257 /* A relative timeout cannot be restarted. */ 3258 if (error == ERESTART && timeout != NULL && 3259 (timeout->_flags & UMTX_ABSTIME) == 0) 3260 error = EINTR; 3261 } 3262 umtxq_unlock(&uq->uq_key); 3263 out: 3264 umtx_key_release(&uq->uq_key); 3265 return (error); 3266 } 3267 3268 /* 3269 * Signal a userland semaphore. 3270 */ 3271 static int 3272 do_sem_wake(struct thread *td, struct _usem *sem) 3273 { 3274 struct umtx_key key; 3275 int error, cnt; 3276 uint32_t flags; 3277 3278 error = fueword32(&sem->_flags, &flags); 3279 if (error == -1) 3280 return (EFAULT); 3281 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) 3282 return (error); 3283 umtxq_lock(&key); 3284 umtxq_busy(&key); 3285 cnt = umtxq_count(&key); 3286 if (cnt > 0) { 3287 /* 3288 * Check if count is greater than 0, this means the memory is 3289 * still being referenced by user code, so we can safely 3290 * update _has_waiters flag. 3291 */ 3292 if (cnt == 1) { 3293 umtxq_unlock(&key); 3294 error = suword32(&sem->_has_waiters, 0); 3295 umtxq_lock(&key); 3296 if (error == -1) 3297 error = EFAULT; 3298 } 3299 umtxq_signal(&key, 1); 3300 } 3301 umtxq_unbusy(&key); 3302 umtxq_unlock(&key); 3303 umtx_key_release(&key); 3304 return (error); 3305 } 3306 #endif 3307 3308 static int 3309 do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) 3310 { 3311 struct abs_timeout timo; 3312 struct umtx_q *uq; 3313 uint32_t count, flags; 3314 int error, rv; 3315 3316 uq = td->td_umtxq; 3317 flags = fuword32(&sem->_flags); 3318 if (timeout != NULL) 3319 abs_timeout_init2(&timo, timeout); 3320 3321 again: 3322 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); 3323 if (error != 0) 3324 return (error); 3325 umtxq_lock(&uq->uq_key); 3326 umtxq_busy(&uq->uq_key); 3327 umtxq_insert(uq); 3328 umtxq_unlock(&uq->uq_key); 3329 rv = fueword32(&sem->_count, &count); 3330 if (rv == -1) { 3331 umtxq_lock(&uq->uq_key); 3332 umtxq_unbusy(&uq->uq_key); 3333 umtxq_remove(uq); 3334 umtxq_unlock(&uq->uq_key); 3335 umtx_key_release(&uq->uq_key); 3336 return (EFAULT); 3337 } 3338 for (;;) { 3339 if (USEM_COUNT(count) != 0) { 3340 umtxq_lock(&uq->uq_key); 3341 umtxq_unbusy(&uq->uq_key); 3342 umtxq_remove(uq); 3343 umtxq_unlock(&uq->uq_key); 3344 umtx_key_release(&uq->uq_key); 3345 return (0); 3346 } 3347 if (count == USEM_HAS_WAITERS) 3348 break; 3349 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); 3350 if (rv == 0) 3351 break; 3352 umtxq_lock(&uq->uq_key); 3353 umtxq_unbusy(&uq->uq_key); 3354 umtxq_remove(uq); 3355 umtxq_unlock(&uq->uq_key); 3356 umtx_key_release(&uq->uq_key); 3357 if (rv == -1) 3358 return (EFAULT); 3359 rv = umtxq_check_susp(td, true); 3360 if (rv != 0) 3361 return (rv); 3362 goto again; 3363 } 3364 umtxq_lock(&uq->uq_key); 3365 umtxq_unbusy(&uq->uq_key); 3366 3367 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); 3368 3369 if ((uq->uq_flags & UQF_UMTXQ) == 0) 3370 error = 0; 3371 else { 3372 umtxq_remove(uq); 3373 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) { 3374 /* A relative timeout cannot be restarted. */ 3375 if (error == ERESTART) 3376 error = EINTR; 3377 if (error == EINTR) { 3378 abs_timeout_update(&timo); 3379 timespecsub(&timo.end, &timo.cur, 3380 &timeout->_timeout); 3381 } 3382 } 3383 } 3384 umtxq_unlock(&uq->uq_key); 3385 umtx_key_release(&uq->uq_key); 3386 return (error); 3387 } 3388 3389 /* 3390 * Signal a userland semaphore. 3391 */ 3392 static int 3393 do_sem2_wake(struct thread *td, struct _usem2 *sem) 3394 { 3395 struct umtx_key key; 3396 int error, cnt, rv; 3397 uint32_t count, flags; 3398 3399 rv = fueword32(&sem->_flags, &flags); 3400 if (rv == -1) 3401 return (EFAULT); 3402 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) 3403 return (error); 3404 umtxq_lock(&key); 3405 umtxq_busy(&key); 3406 cnt = umtxq_count(&key); 3407 if (cnt > 0) { 3408 /* 3409 * If this was the last sleeping thread, clear the waiters 3410 * flag in _count. 3411 */ 3412 if (cnt == 1) { 3413 umtxq_unlock(&key); 3414 rv = fueword32(&sem->_count, &count); 3415 while (rv != -1 && count & USEM_HAS_WAITERS) { 3416 rv = casueword32(&sem->_count, count, &count, 3417 count & ~USEM_HAS_WAITERS); 3418 if (rv == 1) { 3419 rv = umtxq_check_susp(td, true); 3420 if (rv != 0) 3421 break; 3422 } 3423 } 3424 if (rv == -1) 3425 error = EFAULT; 3426 else if (rv > 0) { 3427 error = rv; 3428 } 3429 umtxq_lock(&key); 3430 } 3431 3432 umtxq_signal(&key, 1); 3433 } 3434 umtxq_unbusy(&key); 3435 umtxq_unlock(&key); 3436 umtx_key_release(&key); 3437 return (error); 3438 } 3439 3440 inline int 3441 umtx_copyin_timeout(const void *addr, struct timespec *tsp) 3442 { 3443 int error; 3444 3445 error = copyin(addr, tsp, sizeof(struct timespec)); 3446 if (error == 0) { 3447 if (tsp->tv_sec < 0 || 3448 tsp->tv_nsec >= 1000000000 || 3449 tsp->tv_nsec < 0) 3450 error = EINVAL; 3451 } 3452 return (error); 3453 } 3454 3455 static inline int 3456 umtx_copyin_umtx_time(const void *addr, size_t size, struct _umtx_time *tp) 3457 { 3458 int error; 3459 3460 if (size <= sizeof(struct timespec)) { 3461 tp->_clockid = CLOCK_REALTIME; 3462 tp->_flags = 0; 3463 error = copyin(addr, &tp->_timeout, sizeof(struct timespec)); 3464 } else 3465 error = copyin(addr, tp, sizeof(struct _umtx_time)); 3466 if (error != 0) 3467 return (error); 3468 if (tp->_timeout.tv_sec < 0 || 3469 tp->_timeout.tv_nsec >= 1000000000 || tp->_timeout.tv_nsec < 0) 3470 return (EINVAL); 3471 return (0); 3472 } 3473 3474 static int 3475 __umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap) 3476 { 3477 3478 return (EOPNOTSUPP); 3479 } 3480 3481 static int 3482 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) 3483 { 3484 struct _umtx_time timeout, *tm_p; 3485 int error; 3486 3487 if (uap->uaddr2 == NULL) 3488 tm_p = NULL; 3489 else { 3490 error = umtx_copyin_umtx_time( 3491 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3492 if (error != 0) 3493 return (error); 3494 tm_p = &timeout; 3495 } 3496 return (do_wait(td, uap->obj, uap->val, tm_p, 0, 0)); 3497 } 3498 3499 static int 3500 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap) 3501 { 3502 struct _umtx_time timeout, *tm_p; 3503 int error; 3504 3505 if (uap->uaddr2 == NULL) 3506 tm_p = NULL; 3507 else { 3508 error = umtx_copyin_umtx_time( 3509 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3510 if (error != 0) 3511 return (error); 3512 tm_p = &timeout; 3513 } 3514 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); 3515 } 3516 3517 static int 3518 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) 3519 { 3520 struct _umtx_time *tm_p, timeout; 3521 int error; 3522 3523 if (uap->uaddr2 == NULL) 3524 tm_p = NULL; 3525 else { 3526 error = umtx_copyin_umtx_time( 3527 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3528 if (error != 0) 3529 return (error); 3530 tm_p = &timeout; 3531 } 3532 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); 3533 } 3534 3535 static int 3536 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) 3537 { 3538 3539 return (kern_umtx_wake(td, uap->obj, uap->val, 0)); 3540 } 3541 3542 #define BATCH_SIZE 128 3543 static int 3544 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap) 3545 { 3546 char *uaddrs[BATCH_SIZE], **upp; 3547 int count, error, i, pos, tocopy; 3548 3549 upp = (char **)uap->obj; 3550 error = 0; 3551 for (count = uap->val, pos = 0; count > 0; count -= tocopy, 3552 pos += tocopy) { 3553 tocopy = MIN(count, BATCH_SIZE); 3554 error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *)); 3555 if (error != 0) 3556 break; 3557 for (i = 0; i < tocopy; ++i) 3558 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1); 3559 maybe_yield(); 3560 } 3561 return (error); 3562 } 3563 3564 static int 3565 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) 3566 { 3567 3568 return (kern_umtx_wake(td, uap->obj, uap->val, 1)); 3569 } 3570 3571 static int 3572 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) 3573 { 3574 struct _umtx_time *tm_p, timeout; 3575 int error; 3576 3577 /* Allow a null timespec (wait forever). */ 3578 if (uap->uaddr2 == NULL) 3579 tm_p = NULL; 3580 else { 3581 error = umtx_copyin_umtx_time( 3582 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3583 if (error != 0) 3584 return (error); 3585 tm_p = &timeout; 3586 } 3587 return (do_lock_umutex(td, uap->obj, tm_p, 0)); 3588 } 3589 3590 static int 3591 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) 3592 { 3593 3594 return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY)); 3595 } 3596 3597 static int 3598 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) 3599 { 3600 struct _umtx_time *tm_p, timeout; 3601 int error; 3602 3603 /* Allow a null timespec (wait forever). */ 3604 if (uap->uaddr2 == NULL) 3605 tm_p = NULL; 3606 else { 3607 error = umtx_copyin_umtx_time( 3608 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3609 if (error != 0) 3610 return (error); 3611 tm_p = &timeout; 3612 } 3613 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); 3614 } 3615 3616 static int 3617 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) 3618 { 3619 3620 return (do_wake_umutex(td, uap->obj)); 3621 } 3622 3623 static int 3624 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) 3625 { 3626 3627 return (do_unlock_umutex(td, uap->obj, false)); 3628 } 3629 3630 static int 3631 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) 3632 { 3633 3634 return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1)); 3635 } 3636 3637 static int 3638 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) 3639 { 3640 struct timespec *ts, timeout; 3641 int error; 3642 3643 /* Allow a null timespec (wait forever). */ 3644 if (uap->uaddr2 == NULL) 3645 ts = NULL; 3646 else { 3647 error = umtx_copyin_timeout(uap->uaddr2, &timeout); 3648 if (error != 0) 3649 return (error); 3650 ts = &timeout; 3651 } 3652 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 3653 } 3654 3655 static int 3656 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) 3657 { 3658 3659 return (do_cv_signal(td, uap->obj)); 3660 } 3661 3662 static int 3663 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) 3664 { 3665 3666 return (do_cv_broadcast(td, uap->obj)); 3667 } 3668 3669 static int 3670 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap) 3671 { 3672 struct _umtx_time timeout; 3673 int error; 3674 3675 /* Allow a null timespec (wait forever). */ 3676 if (uap->uaddr2 == NULL) { 3677 error = do_rw_rdlock(td, uap->obj, uap->val, 0); 3678 } else { 3679 error = umtx_copyin_umtx_time(uap->uaddr2, 3680 (size_t)uap->uaddr1, &timeout); 3681 if (error != 0) 3682 return (error); 3683 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); 3684 } 3685 return (error); 3686 } 3687 3688 static int 3689 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap) 3690 { 3691 struct _umtx_time timeout; 3692 int error; 3693 3694 /* Allow a null timespec (wait forever). */ 3695 if (uap->uaddr2 == NULL) { 3696 error = do_rw_wrlock(td, uap->obj, 0); 3697 } else { 3698 error = umtx_copyin_umtx_time(uap->uaddr2, 3699 (size_t)uap->uaddr1, &timeout); 3700 if (error != 0) 3701 return (error); 3702 3703 error = do_rw_wrlock(td, uap->obj, &timeout); 3704 } 3705 return (error); 3706 } 3707 3708 static int 3709 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) 3710 { 3711 3712 return (do_rw_unlock(td, uap->obj)); 3713 } 3714 3715 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 3716 static int 3717 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap) 3718 { 3719 struct _umtx_time *tm_p, timeout; 3720 int error; 3721 3722 /* Allow a null timespec (wait forever). */ 3723 if (uap->uaddr2 == NULL) 3724 tm_p = NULL; 3725 else { 3726 error = umtx_copyin_umtx_time( 3727 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3728 if (error != 0) 3729 return (error); 3730 tm_p = &timeout; 3731 } 3732 return (do_sem_wait(td, uap->obj, tm_p)); 3733 } 3734 3735 static int 3736 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap) 3737 { 3738 3739 return (do_sem_wake(td, uap->obj)); 3740 } 3741 #endif 3742 3743 static int 3744 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap) 3745 { 3746 3747 return (do_wake2_umutex(td, uap->obj, uap->val)); 3748 } 3749 3750 static int 3751 __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap) 3752 { 3753 struct _umtx_time *tm_p, timeout; 3754 size_t uasize; 3755 int error; 3756 3757 /* Allow a null timespec (wait forever). */ 3758 if (uap->uaddr2 == NULL) { 3759 uasize = 0; 3760 tm_p = NULL; 3761 } else { 3762 uasize = (size_t)uap->uaddr1; 3763 error = umtx_copyin_umtx_time(uap->uaddr2, uasize, &timeout); 3764 if (error != 0) 3765 return (error); 3766 tm_p = &timeout; 3767 } 3768 error = do_sem2_wait(td, uap->obj, tm_p); 3769 if (error == EINTR && uap->uaddr2 != NULL && 3770 (timeout._flags & UMTX_ABSTIME) == 0 && 3771 uasize >= sizeof(struct _umtx_time) + sizeof(struct timespec)) { 3772 error = copyout(&timeout._timeout, 3773 (struct _umtx_time *)uap->uaddr2 + 1, 3774 sizeof(struct timespec)); 3775 if (error == 0) { 3776 error = EINTR; 3777 } 3778 } 3779 3780 return (error); 3781 } 3782 3783 static int 3784 __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap) 3785 { 3786 3787 return (do_sem2_wake(td, uap->obj)); 3788 } 3789 3790 #define USHM_OBJ_UMTX(o) \ 3791 ((struct umtx_shm_obj_list *)(&(o)->umtx_data)) 3792 3793 #define USHMF_REG_LINKED 0x0001 3794 #define USHMF_OBJ_LINKED 0x0002 3795 struct umtx_shm_reg { 3796 TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link; 3797 LIST_ENTRY(umtx_shm_reg) ushm_obj_link; 3798 struct umtx_key ushm_key; 3799 struct ucred *ushm_cred; 3800 struct shmfd *ushm_obj; 3801 u_int ushm_refcnt; 3802 u_int ushm_flags; 3803 }; 3804 3805 LIST_HEAD(umtx_shm_obj_list, umtx_shm_reg); 3806 TAILQ_HEAD(umtx_shm_reg_head, umtx_shm_reg); 3807 3808 static uma_zone_t umtx_shm_reg_zone; 3809 static struct umtx_shm_reg_head umtx_shm_registry[UMTX_CHAINS]; 3810 static struct mtx umtx_shm_lock; 3811 static struct umtx_shm_reg_head umtx_shm_reg_delfree = 3812 TAILQ_HEAD_INITIALIZER(umtx_shm_reg_delfree); 3813 3814 static void umtx_shm_free_reg(struct umtx_shm_reg *reg); 3815 3816 static void 3817 umtx_shm_reg_delfree_tq(void *context __unused, int pending __unused) 3818 { 3819 struct umtx_shm_reg_head d; 3820 struct umtx_shm_reg *reg, *reg1; 3821 3822 TAILQ_INIT(&d); 3823 mtx_lock(&umtx_shm_lock); 3824 TAILQ_CONCAT(&d, &umtx_shm_reg_delfree, ushm_reg_link); 3825 mtx_unlock(&umtx_shm_lock); 3826 TAILQ_FOREACH_SAFE(reg, &d, ushm_reg_link, reg1) { 3827 TAILQ_REMOVE(&d, reg, ushm_reg_link); 3828 umtx_shm_free_reg(reg); 3829 } 3830 } 3831 3832 static struct task umtx_shm_reg_delfree_task = 3833 TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL); 3834 3835 static struct umtx_shm_reg * 3836 umtx_shm_find_reg_locked(const struct umtx_key *key) 3837 { 3838 struct umtx_shm_reg *reg; 3839 struct umtx_shm_reg_head *reg_head; 3840 3841 KASSERT(key->shared, ("umtx_p_find_rg: private key")); 3842 mtx_assert(&umtx_shm_lock, MA_OWNED); 3843 reg_head = &umtx_shm_registry[key->hash]; 3844 TAILQ_FOREACH(reg, reg_head, ushm_reg_link) { 3845 KASSERT(reg->ushm_key.shared, 3846 ("non-shared key on reg %p %d", reg, reg->ushm_key.shared)); 3847 if (reg->ushm_key.info.shared.object == 3848 key->info.shared.object && 3849 reg->ushm_key.info.shared.offset == 3850 key->info.shared.offset) { 3851 KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM")); 3852 KASSERT(reg->ushm_refcnt > 0, 3853 ("reg %p refcnt 0 onlist", reg)); 3854 KASSERT((reg->ushm_flags & USHMF_REG_LINKED) != 0, 3855 ("reg %p not linked", reg)); 3856 reg->ushm_refcnt++; 3857 return (reg); 3858 } 3859 } 3860 return (NULL); 3861 } 3862 3863 static struct umtx_shm_reg * 3864 umtx_shm_find_reg(const struct umtx_key *key) 3865 { 3866 struct umtx_shm_reg *reg; 3867 3868 mtx_lock(&umtx_shm_lock); 3869 reg = umtx_shm_find_reg_locked(key); 3870 mtx_unlock(&umtx_shm_lock); 3871 return (reg); 3872 } 3873 3874 static void 3875 umtx_shm_free_reg(struct umtx_shm_reg *reg) 3876 { 3877 3878 chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0); 3879 crfree(reg->ushm_cred); 3880 shm_drop(reg->ushm_obj); 3881 uma_zfree(umtx_shm_reg_zone, reg); 3882 } 3883 3884 static bool 3885 umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool force) 3886 { 3887 bool res; 3888 3889 mtx_assert(&umtx_shm_lock, MA_OWNED); 3890 KASSERT(reg->ushm_refcnt > 0, ("ushm_reg %p refcnt 0", reg)); 3891 reg->ushm_refcnt--; 3892 res = reg->ushm_refcnt == 0; 3893 if (res || force) { 3894 if ((reg->ushm_flags & USHMF_REG_LINKED) != 0) { 3895 TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], 3896 reg, ushm_reg_link); 3897 reg->ushm_flags &= ~USHMF_REG_LINKED; 3898 } 3899 if ((reg->ushm_flags & USHMF_OBJ_LINKED) != 0) { 3900 LIST_REMOVE(reg, ushm_obj_link); 3901 reg->ushm_flags &= ~USHMF_OBJ_LINKED; 3902 } 3903 } 3904 return (res); 3905 } 3906 3907 static void 3908 umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool force) 3909 { 3910 vm_object_t object; 3911 bool dofree; 3912 3913 if (force) { 3914 object = reg->ushm_obj->shm_object; 3915 VM_OBJECT_WLOCK(object); 3916 object->flags |= OBJ_UMTXDEAD; 3917 VM_OBJECT_WUNLOCK(object); 3918 } 3919 mtx_lock(&umtx_shm_lock); 3920 dofree = umtx_shm_unref_reg_locked(reg, force); 3921 mtx_unlock(&umtx_shm_lock); 3922 if (dofree) 3923 umtx_shm_free_reg(reg); 3924 } 3925 3926 void 3927 umtx_shm_object_init(vm_object_t object) 3928 { 3929 3930 LIST_INIT(USHM_OBJ_UMTX(object)); 3931 } 3932 3933 void 3934 umtx_shm_object_terminated(vm_object_t object) 3935 { 3936 struct umtx_shm_reg *reg, *reg1; 3937 bool dofree; 3938 3939 if (LIST_EMPTY(USHM_OBJ_UMTX(object))) 3940 return; 3941 3942 dofree = false; 3943 mtx_lock(&umtx_shm_lock); 3944 LIST_FOREACH_SAFE(reg, USHM_OBJ_UMTX(object), ushm_obj_link, reg1) { 3945 if (umtx_shm_unref_reg_locked(reg, true)) { 3946 TAILQ_INSERT_TAIL(&umtx_shm_reg_delfree, reg, 3947 ushm_reg_link); 3948 dofree = true; 3949 } 3950 } 3951 mtx_unlock(&umtx_shm_lock); 3952 if (dofree) 3953 taskqueue_enqueue(taskqueue_thread, &umtx_shm_reg_delfree_task); 3954 } 3955 3956 static int 3957 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key, 3958 struct umtx_shm_reg **res) 3959 { 3960 struct umtx_shm_reg *reg, *reg1; 3961 struct ucred *cred; 3962 int error; 3963 3964 reg = umtx_shm_find_reg(key); 3965 if (reg != NULL) { 3966 *res = reg; 3967 return (0); 3968 } 3969 cred = td->td_ucred; 3970 if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP))) 3971 return (ENOMEM); 3972 reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO); 3973 reg->ushm_refcnt = 1; 3974 bcopy(key, ®->ushm_key, sizeof(*key)); 3975 reg->ushm_obj = shm_alloc(td->td_ucred, O_RDWR); 3976 reg->ushm_cred = crhold(cred); 3977 error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE); 3978 if (error != 0) { 3979 umtx_shm_free_reg(reg); 3980 return (error); 3981 } 3982 mtx_lock(&umtx_shm_lock); 3983 reg1 = umtx_shm_find_reg_locked(key); 3984 if (reg1 != NULL) { 3985 mtx_unlock(&umtx_shm_lock); 3986 umtx_shm_free_reg(reg); 3987 *res = reg1; 3988 return (0); 3989 } 3990 reg->ushm_refcnt++; 3991 TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link); 3992 LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg, 3993 ushm_obj_link); 3994 reg->ushm_flags = USHMF_REG_LINKED | USHMF_OBJ_LINKED; 3995 mtx_unlock(&umtx_shm_lock); 3996 *res = reg; 3997 return (0); 3998 } 3999 4000 static int 4001 umtx_shm_alive(struct thread *td, void *addr) 4002 { 4003 vm_map_t map; 4004 vm_map_entry_t entry; 4005 vm_object_t object; 4006 vm_pindex_t pindex; 4007 vm_prot_t prot; 4008 int res, ret; 4009 boolean_t wired; 4010 4011 map = &td->td_proc->p_vmspace->vm_map; 4012 res = vm_map_lookup(&map, (uintptr_t)addr, VM_PROT_READ, &entry, 4013 &object, &pindex, &prot, &wired); 4014 if (res != KERN_SUCCESS) 4015 return (EFAULT); 4016 if (object == NULL) 4017 ret = EINVAL; 4018 else 4019 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0; 4020 vm_map_lookup_done(map, entry); 4021 return (ret); 4022 } 4023 4024 static void 4025 umtx_shm_init(void) 4026 { 4027 int i; 4028 4029 umtx_shm_reg_zone = uma_zcreate("umtx_shm", sizeof(struct umtx_shm_reg), 4030 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 4031 mtx_init(&umtx_shm_lock, "umtxshm", NULL, MTX_DEF); 4032 for (i = 0; i < nitems(umtx_shm_registry); i++) 4033 TAILQ_INIT(&umtx_shm_registry[i]); 4034 } 4035 4036 static int 4037 umtx_shm(struct thread *td, void *addr, u_int flags) 4038 { 4039 struct umtx_key key; 4040 struct umtx_shm_reg *reg; 4041 struct file *fp; 4042 int error, fd; 4043 4044 if (__bitcount(flags & (UMTX_SHM_CREAT | UMTX_SHM_LOOKUP | 4045 UMTX_SHM_DESTROY| UMTX_SHM_ALIVE)) != 1) 4046 return (EINVAL); 4047 if ((flags & UMTX_SHM_ALIVE) != 0) 4048 return (umtx_shm_alive(td, addr)); 4049 error = umtx_key_get(addr, TYPE_SHM, PROCESS_SHARE, &key); 4050 if (error != 0) 4051 return (error); 4052 KASSERT(key.shared == 1, ("non-shared key")); 4053 if ((flags & UMTX_SHM_CREAT) != 0) { 4054 error = umtx_shm_create_reg(td, &key, ®); 4055 } else { 4056 reg = umtx_shm_find_reg(&key); 4057 if (reg == NULL) 4058 error = ESRCH; 4059 } 4060 umtx_key_release(&key); 4061 if (error != 0) 4062 return (error); 4063 KASSERT(reg != NULL, ("no reg")); 4064 if ((flags & UMTX_SHM_DESTROY) != 0) { 4065 umtx_shm_unref_reg(reg, true); 4066 } else { 4067 #if 0 4068 #ifdef MAC 4069 error = mac_posixshm_check_open(td->td_ucred, 4070 reg->ushm_obj, FFLAGS(O_RDWR)); 4071 if (error == 0) 4072 #endif 4073 error = shm_access(reg->ushm_obj, td->td_ucred, 4074 FFLAGS(O_RDWR)); 4075 if (error == 0) 4076 #endif 4077 error = falloc_caps(td, &fp, &fd, O_CLOEXEC, NULL); 4078 if (error == 0) { 4079 shm_hold(reg->ushm_obj); 4080 finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj, 4081 &shm_ops); 4082 td->td_retval[0] = fd; 4083 fdrop(fp, td); 4084 } 4085 } 4086 umtx_shm_unref_reg(reg, false); 4087 return (error); 4088 } 4089 4090 static int 4091 __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap) 4092 { 4093 4094 return (umtx_shm(td, uap->uaddr1, uap->val)); 4095 } 4096 4097 static int 4098 umtx_robust_lists(struct thread *td, struct umtx_robust_lists_params *rbp) 4099 { 4100 4101 td->td_rb_list = rbp->robust_list_offset; 4102 td->td_rbp_list = rbp->robust_priv_list_offset; 4103 td->td_rb_inact = rbp->robust_inact_offset; 4104 return (0); 4105 } 4106 4107 static int 4108 __umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap) 4109 { 4110 struct umtx_robust_lists_params rb; 4111 int error; 4112 4113 if (uap->val > sizeof(rb)) 4114 return (EINVAL); 4115 bzero(&rb, sizeof(rb)); 4116 error = copyin(uap->uaddr1, &rb, uap->val); 4117 if (error != 0) 4118 return (error); 4119 return (umtx_robust_lists(td, &rb)); 4120 } 4121 4122 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); 4123 4124 static const _umtx_op_func op_table[] = { 4125 [UMTX_OP_RESERVED0] = __umtx_op_unimpl, 4126 [UMTX_OP_RESERVED1] = __umtx_op_unimpl, 4127 [UMTX_OP_WAIT] = __umtx_op_wait, 4128 [UMTX_OP_WAKE] = __umtx_op_wake, 4129 [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, 4130 [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex, 4131 [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, 4132 [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, 4133 [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait, 4134 [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, 4135 [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast, 4136 [UMTX_OP_WAIT_UINT] = __umtx_op_wait_uint, 4137 [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock, 4138 [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock, 4139 [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock, 4140 [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private, 4141 [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private, 4142 [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex, 4143 [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex, 4144 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 4145 [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait, 4146 [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake, 4147 #else 4148 [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl, 4149 [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl, 4150 #endif 4151 [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private, 4152 [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex, 4153 [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait, 4154 [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, 4155 [UMTX_OP_SHM] = __umtx_op_shm, 4156 [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists, 4157 }; 4158 4159 int 4160 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap) 4161 { 4162 4163 if ((unsigned)uap->op < nitems(op_table)) 4164 return (*op_table[uap->op])(td, uap); 4165 return (EINVAL); 4166 } 4167 4168 #ifdef COMPAT_FREEBSD32 4169 4170 struct timespec32 { 4171 int32_t tv_sec; 4172 int32_t tv_nsec; 4173 }; 4174 4175 struct umtx_time32 { 4176 struct timespec32 timeout; 4177 uint32_t flags; 4178 uint32_t clockid; 4179 }; 4180 4181 static inline int 4182 umtx_copyin_timeout32(void *addr, struct timespec *tsp) 4183 { 4184 struct timespec32 ts32; 4185 int error; 4186 4187 error = copyin(addr, &ts32, sizeof(struct timespec32)); 4188 if (error == 0) { 4189 if (ts32.tv_sec < 0 || 4190 ts32.tv_nsec >= 1000000000 || 4191 ts32.tv_nsec < 0) 4192 error = EINVAL; 4193 else { 4194 tsp->tv_sec = ts32.tv_sec; 4195 tsp->tv_nsec = ts32.tv_nsec; 4196 } 4197 } 4198 return (error); 4199 } 4200 4201 static inline int 4202 umtx_copyin_umtx_time32(const void *addr, size_t size, struct _umtx_time *tp) 4203 { 4204 struct umtx_time32 t32; 4205 int error; 4206 4207 t32.clockid = CLOCK_REALTIME; 4208 t32.flags = 0; 4209 if (size <= sizeof(struct timespec32)) 4210 error = copyin(addr, &t32.timeout, sizeof(struct timespec32)); 4211 else 4212 error = copyin(addr, &t32, sizeof(struct umtx_time32)); 4213 if (error != 0) 4214 return (error); 4215 if (t32.timeout.tv_sec < 0 || 4216 t32.timeout.tv_nsec >= 1000000000 || t32.timeout.tv_nsec < 0) 4217 return (EINVAL); 4218 tp->_timeout.tv_sec = t32.timeout.tv_sec; 4219 tp->_timeout.tv_nsec = t32.timeout.tv_nsec; 4220 tp->_flags = t32.flags; 4221 tp->_clockid = t32.clockid; 4222 return (0); 4223 } 4224 4225 static int 4226 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4227 { 4228 struct _umtx_time *tm_p, timeout; 4229 int error; 4230 4231 if (uap->uaddr2 == NULL) 4232 tm_p = NULL; 4233 else { 4234 error = umtx_copyin_umtx_time32(uap->uaddr2, 4235 (size_t)uap->uaddr1, &timeout); 4236 if (error != 0) 4237 return (error); 4238 tm_p = &timeout; 4239 } 4240 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); 4241 } 4242 4243 static int 4244 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 4245 { 4246 struct _umtx_time *tm_p, timeout; 4247 int error; 4248 4249 /* Allow a null timespec (wait forever). */ 4250 if (uap->uaddr2 == NULL) 4251 tm_p = NULL; 4252 else { 4253 error = umtx_copyin_umtx_time32(uap->uaddr2, 4254 (size_t)uap->uaddr1, &timeout); 4255 if (error != 0) 4256 return (error); 4257 tm_p = &timeout; 4258 } 4259 return (do_lock_umutex(td, uap->obj, tm_p, 0)); 4260 } 4261 4262 static int 4263 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 4264 { 4265 struct _umtx_time *tm_p, timeout; 4266 int error; 4267 4268 /* Allow a null timespec (wait forever). */ 4269 if (uap->uaddr2 == NULL) 4270 tm_p = NULL; 4271 else { 4272 error = umtx_copyin_umtx_time32(uap->uaddr2, 4273 (size_t)uap->uaddr1, &timeout); 4274 if (error != 0) 4275 return (error); 4276 tm_p = &timeout; 4277 } 4278 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); 4279 } 4280 4281 static int 4282 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4283 { 4284 struct timespec *ts, timeout; 4285 int error; 4286 4287 /* Allow a null timespec (wait forever). */ 4288 if (uap->uaddr2 == NULL) 4289 ts = NULL; 4290 else { 4291 error = umtx_copyin_timeout32(uap->uaddr2, &timeout); 4292 if (error != 0) 4293 return (error); 4294 ts = &timeout; 4295 } 4296 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 4297 } 4298 4299 static int 4300 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap) 4301 { 4302 struct _umtx_time timeout; 4303 int error; 4304 4305 /* Allow a null timespec (wait forever). */ 4306 if (uap->uaddr2 == NULL) { 4307 error = do_rw_rdlock(td, uap->obj, uap->val, 0); 4308 } else { 4309 error = umtx_copyin_umtx_time32(uap->uaddr2, 4310 (size_t)uap->uaddr1, &timeout); 4311 if (error != 0) 4312 return (error); 4313 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); 4314 } 4315 return (error); 4316 } 4317 4318 static int 4319 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap) 4320 { 4321 struct _umtx_time timeout; 4322 int error; 4323 4324 /* Allow a null timespec (wait forever). */ 4325 if (uap->uaddr2 == NULL) { 4326 error = do_rw_wrlock(td, uap->obj, 0); 4327 } else { 4328 error = umtx_copyin_umtx_time32(uap->uaddr2, 4329 (size_t)uap->uaddr1, &timeout); 4330 if (error != 0) 4331 return (error); 4332 error = do_rw_wrlock(td, uap->obj, &timeout); 4333 } 4334 return (error); 4335 } 4336 4337 static int 4338 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap) 4339 { 4340 struct _umtx_time *tm_p, timeout; 4341 int error; 4342 4343 if (uap->uaddr2 == NULL) 4344 tm_p = NULL; 4345 else { 4346 error = umtx_copyin_umtx_time32( 4347 uap->uaddr2, (size_t)uap->uaddr1,&timeout); 4348 if (error != 0) 4349 return (error); 4350 tm_p = &timeout; 4351 } 4352 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); 4353 } 4354 4355 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 4356 static int 4357 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4358 { 4359 struct _umtx_time *tm_p, timeout; 4360 int error; 4361 4362 /* Allow a null timespec (wait forever). */ 4363 if (uap->uaddr2 == NULL) 4364 tm_p = NULL; 4365 else { 4366 error = umtx_copyin_umtx_time32(uap->uaddr2, 4367 (size_t)uap->uaddr1, &timeout); 4368 if (error != 0) 4369 return (error); 4370 tm_p = &timeout; 4371 } 4372 return (do_sem_wait(td, uap->obj, tm_p)); 4373 } 4374 #endif 4375 4376 static int 4377 __umtx_op_sem2_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4378 { 4379 struct _umtx_time *tm_p, timeout; 4380 size_t uasize; 4381 int error; 4382 4383 /* Allow a null timespec (wait forever). */ 4384 if (uap->uaddr2 == NULL) { 4385 uasize = 0; 4386 tm_p = NULL; 4387 } else { 4388 uasize = (size_t)uap->uaddr1; 4389 error = umtx_copyin_umtx_time32(uap->uaddr2, uasize, &timeout); 4390 if (error != 0) 4391 return (error); 4392 tm_p = &timeout; 4393 } 4394 error = do_sem2_wait(td, uap->obj, tm_p); 4395 if (error == EINTR && uap->uaddr2 != NULL && 4396 (timeout._flags & UMTX_ABSTIME) == 0 && 4397 uasize >= sizeof(struct umtx_time32) + sizeof(struct timespec32)) { 4398 struct timespec32 remain32 = { 4399 .tv_sec = timeout._timeout.tv_sec, 4400 .tv_nsec = timeout._timeout.tv_nsec 4401 }; 4402 error = copyout(&remain32, 4403 (struct umtx_time32 *)uap->uaddr2 + 1, 4404 sizeof(struct timespec32)); 4405 if (error == 0) { 4406 error = EINTR; 4407 } 4408 } 4409 4410 return (error); 4411 } 4412 4413 static int 4414 __umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap) 4415 { 4416 uint32_t uaddrs[BATCH_SIZE], **upp; 4417 int count, error, i, pos, tocopy; 4418 4419 upp = (uint32_t **)uap->obj; 4420 error = 0; 4421 for (count = uap->val, pos = 0; count > 0; count -= tocopy, 4422 pos += tocopy) { 4423 tocopy = MIN(count, BATCH_SIZE); 4424 error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t)); 4425 if (error != 0) 4426 break; 4427 for (i = 0; i < tocopy; ++i) 4428 kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i], 4429 INT_MAX, 1); 4430 maybe_yield(); 4431 } 4432 return (error); 4433 } 4434 4435 struct umtx_robust_lists_params_compat32 { 4436 uint32_t robust_list_offset; 4437 uint32_t robust_priv_list_offset; 4438 uint32_t robust_inact_offset; 4439 }; 4440 4441 static int 4442 __umtx_op_robust_lists_compat32(struct thread *td, struct _umtx_op_args *uap) 4443 { 4444 struct umtx_robust_lists_params rb; 4445 struct umtx_robust_lists_params_compat32 rb32; 4446 int error; 4447 4448 if (uap->val > sizeof(rb32)) 4449 return (EINVAL); 4450 bzero(&rb, sizeof(rb)); 4451 bzero(&rb32, sizeof(rb32)); 4452 error = copyin(uap->uaddr1, &rb32, uap->val); 4453 if (error != 0) 4454 return (error); 4455 rb.robust_list_offset = rb32.robust_list_offset; 4456 rb.robust_priv_list_offset = rb32.robust_priv_list_offset; 4457 rb.robust_inact_offset = rb32.robust_inact_offset; 4458 return (umtx_robust_lists(td, &rb)); 4459 } 4460 4461 static const _umtx_op_func op_table_compat32[] = { 4462 [UMTX_OP_RESERVED0] = __umtx_op_unimpl, 4463 [UMTX_OP_RESERVED1] = __umtx_op_unimpl, 4464 [UMTX_OP_WAIT] = __umtx_op_wait_compat32, 4465 [UMTX_OP_WAKE] = __umtx_op_wake, 4466 [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, 4467 [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex_compat32, 4468 [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, 4469 [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, 4470 [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait_compat32, 4471 [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, 4472 [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast, 4473 [UMTX_OP_WAIT_UINT] = __umtx_op_wait_compat32, 4474 [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock_compat32, 4475 [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock_compat32, 4476 [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock, 4477 [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private_compat32, 4478 [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private, 4479 [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex_compat32, 4480 [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex, 4481 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 4482 [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait_compat32, 4483 [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake, 4484 #else 4485 [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl, 4486 [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl, 4487 #endif 4488 [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private32, 4489 [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex, 4490 [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait_compat32, 4491 [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, 4492 [UMTX_OP_SHM] = __umtx_op_shm, 4493 [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists_compat32, 4494 }; 4495 4496 int 4497 freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap) 4498 { 4499 4500 if ((unsigned)uap->op < nitems(op_table_compat32)) { 4501 return (*op_table_compat32[uap->op])(td, 4502 (struct _umtx_op_args *)uap); 4503 } 4504 return (EINVAL); 4505 } 4506 #endif 4507 4508 void 4509 umtx_thread_init(struct thread *td) 4510 { 4511 4512 td->td_umtxq = umtxq_alloc(); 4513 td->td_umtxq->uq_thread = td; 4514 } 4515 4516 void 4517 umtx_thread_fini(struct thread *td) 4518 { 4519 4520 umtxq_free(td->td_umtxq); 4521 } 4522 4523 /* 4524 * It will be called when new thread is created, e.g fork(). 4525 */ 4526 void 4527 umtx_thread_alloc(struct thread *td) 4528 { 4529 struct umtx_q *uq; 4530 4531 uq = td->td_umtxq; 4532 uq->uq_inherited_pri = PRI_MAX; 4533 4534 KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); 4535 KASSERT(uq->uq_thread == td, ("uq_thread != td")); 4536 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); 4537 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); 4538 } 4539 4540 /* 4541 * exec() hook. 4542 * 4543 * Clear robust lists for all process' threads, not delaying the 4544 * cleanup to thread_exit hook, since the relevant address space is 4545 * destroyed right now. 4546 */ 4547 static void 4548 umtx_exec_hook(void *arg __unused, struct proc *p, 4549 struct image_params *imgp __unused) 4550 { 4551 struct thread *td; 4552 4553 KASSERT(p == curproc, ("need curproc")); 4554 KASSERT((p->p_flag & P_HADTHREADS) == 0 || 4555 (p->p_flag & P_STOPPED_SINGLE) != 0, 4556 ("curproc must be single-threaded")); 4557 /* 4558 * There is no need to lock the list as only this thread can be 4559 * running. 4560 */ 4561 FOREACH_THREAD_IN_PROC(p, td) { 4562 KASSERT(td == curthread || 4563 ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)), 4564 ("running thread %p %p", p, td)); 4565 umtx_thread_cleanup(td); 4566 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0; 4567 } 4568 } 4569 4570 /* 4571 * thread_exit() hook. 4572 */ 4573 void 4574 umtx_thread_exit(struct thread *td) 4575 { 4576 4577 umtx_thread_cleanup(td); 4578 } 4579 4580 static int 4581 umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res) 4582 { 4583 u_long res1; 4584 #ifdef COMPAT_FREEBSD32 4585 uint32_t res32; 4586 #endif 4587 int error; 4588 4589 #ifdef COMPAT_FREEBSD32 4590 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { 4591 error = fueword32((void *)ptr, &res32); 4592 if (error == 0) 4593 res1 = res32; 4594 } else 4595 #endif 4596 { 4597 error = fueword((void *)ptr, &res1); 4598 } 4599 if (error == 0) 4600 *res = res1; 4601 else 4602 error = EFAULT; 4603 return (error); 4604 } 4605 4606 static void 4607 umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list) 4608 { 4609 #ifdef COMPAT_FREEBSD32 4610 struct umutex32 m32; 4611 4612 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { 4613 memcpy(&m32, m, sizeof(m32)); 4614 *rb_list = m32.m_rb_lnk; 4615 } else 4616 #endif 4617 *rb_list = m->m_rb_lnk; 4618 } 4619 4620 static int 4621 umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact) 4622 { 4623 struct umutex m; 4624 int error; 4625 4626 KASSERT(td->td_proc == curproc, ("need current vmspace")); 4627 error = copyin((void *)rbp, &m, sizeof(m)); 4628 if (error != 0) 4629 return (error); 4630 if (rb_list != NULL) 4631 umtx_read_rb_list(td, &m, rb_list); 4632 if ((m.m_flags & UMUTEX_ROBUST) == 0) 4633 return (EINVAL); 4634 if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid) 4635 /* inact is cleared after unlock, allow the inconsistency */ 4636 return (inact ? 0 : EINVAL); 4637 return (do_unlock_umutex(td, (struct umutex *)rbp, true)); 4638 } 4639 4640 static void 4641 umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact, 4642 const char *name) 4643 { 4644 int error, i; 4645 uintptr_t rbp; 4646 bool inact; 4647 4648 if (rb_list == 0) 4649 return; 4650 error = umtx_read_uptr(td, rb_list, &rbp); 4651 for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) { 4652 if (rbp == *rb_inact) { 4653 inact = true; 4654 *rb_inact = 0; 4655 } else 4656 inact = false; 4657 error = umtx_handle_rb(td, rbp, &rbp, inact); 4658 } 4659 if (i == umtx_max_rb && umtx_verbose_rb) { 4660 uprintf("comm %s pid %d: reached umtx %smax rb %d\n", 4661 td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb); 4662 } 4663 if (error != 0 && umtx_verbose_rb) { 4664 uprintf("comm %s pid %d: handling %srb error %d\n", 4665 td->td_proc->p_comm, td->td_proc->p_pid, name, error); 4666 } 4667 } 4668 4669 /* 4670 * Clean up umtx data. 4671 */ 4672 static void 4673 umtx_thread_cleanup(struct thread *td) 4674 { 4675 struct umtx_q *uq; 4676 struct umtx_pi *pi; 4677 uintptr_t rb_inact; 4678 4679 /* 4680 * Disown pi mutexes. 4681 */ 4682 uq = td->td_umtxq; 4683 if (uq != NULL) { 4684 if (uq->uq_inherited_pri != PRI_MAX || 4685 !TAILQ_EMPTY(&uq->uq_pi_contested)) { 4686 mtx_lock(&umtx_lock); 4687 uq->uq_inherited_pri = PRI_MAX; 4688 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { 4689 pi->pi_owner = NULL; 4690 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); 4691 } 4692 mtx_unlock(&umtx_lock); 4693 } 4694 sched_lend_user_prio_cond(td, PRI_MAX); 4695 } 4696 4697 if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0) 4698 return; 4699 4700 /* 4701 * Handle terminated robust mutexes. Must be done after 4702 * robust pi disown, otherwise unlock could see unowned 4703 * entries. 4704 */ 4705 rb_inact = td->td_rb_inact; 4706 if (rb_inact != 0) 4707 (void)umtx_read_uptr(td, rb_inact, &rb_inact); 4708 umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, ""); 4709 umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv "); 4710 if (rb_inact != 0) 4711 (void)umtx_handle_rb(td, rb_inact, NULL, true); 4712 } 4713