1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2015, 2016 The FreeBSD Foundation 5 * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 6 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 7 * All rights reserved. 8 * 9 * Portions of this software were developed by Konstantin Belousov 10 * under sponsorship from the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice unmodified, this list of conditions, and the following 17 * disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_umtx_profiling.h" 38 39 #include <sys/param.h> 40 #include <sys/kernel.h> 41 #include <sys/fcntl.h> 42 #include <sys/file.h> 43 #include <sys/filedesc.h> 44 #include <sys/limits.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/mutex.h> 49 #include <sys/priv.h> 50 #include <sys/proc.h> 51 #include <sys/resource.h> 52 #include <sys/resourcevar.h> 53 #include <sys/rwlock.h> 54 #include <sys/sbuf.h> 55 #include <sys/sched.h> 56 #include <sys/smp.h> 57 #include <sys/sysctl.h> 58 #include <sys/sysent.h> 59 #include <sys/systm.h> 60 #include <sys/sysproto.h> 61 #include <sys/syscallsubr.h> 62 #include <sys/taskqueue.h> 63 #include <sys/time.h> 64 #include <sys/eventhandler.h> 65 #include <sys/umtx.h> 66 67 #include <security/mac/mac_framework.h> 68 69 #include <vm/vm.h> 70 #include <vm/vm_param.h> 71 #include <vm/pmap.h> 72 #include <vm/vm_map.h> 73 #include <vm/vm_object.h> 74 75 #include <machine/atomic.h> 76 #include <machine/cpu.h> 77 78 #ifdef COMPAT_FREEBSD32 79 #include <compat/freebsd32/freebsd32_proto.h> 80 #endif 81 82 #define _UMUTEX_TRY 1 83 #define _UMUTEX_WAIT 2 84 85 #ifdef UMTX_PROFILING 86 #define UPROF_PERC_BIGGER(w, f, sw, sf) \ 87 (((w) > (sw)) || ((w) == (sw) && (f) > (sf))) 88 #endif 89 90 /* Priority inheritance mutex info. */ 91 struct umtx_pi { 92 /* Owner thread */ 93 struct thread *pi_owner; 94 95 /* Reference count */ 96 int pi_refcount; 97 98 /* List entry to link umtx holding by thread */ 99 TAILQ_ENTRY(umtx_pi) pi_link; 100 101 /* List entry in hash */ 102 TAILQ_ENTRY(umtx_pi) pi_hashlink; 103 104 /* List for waiters */ 105 TAILQ_HEAD(,umtx_q) pi_blocked; 106 107 /* Identify a userland lock object */ 108 struct umtx_key pi_key; 109 }; 110 111 /* A userland synchronous object user. */ 112 struct umtx_q { 113 /* Linked list for the hash. */ 114 TAILQ_ENTRY(umtx_q) uq_link; 115 116 /* Umtx key. */ 117 struct umtx_key uq_key; 118 119 /* Umtx flags. */ 120 int uq_flags; 121 #define UQF_UMTXQ 0x0001 122 123 /* The thread waits on. */ 124 struct thread *uq_thread; 125 126 /* 127 * Blocked on PI mutex. read can use chain lock 128 * or umtx_lock, write must have both chain lock and 129 * umtx_lock being hold. 130 */ 131 struct umtx_pi *uq_pi_blocked; 132 133 /* On blocked list */ 134 TAILQ_ENTRY(umtx_q) uq_lockq; 135 136 /* Thread contending with us */ 137 TAILQ_HEAD(,umtx_pi) uq_pi_contested; 138 139 /* Inherited priority from PP mutex */ 140 u_char uq_inherited_pri; 141 142 /* Spare queue ready to be reused */ 143 struct umtxq_queue *uq_spare_queue; 144 145 /* The queue we on */ 146 struct umtxq_queue *uq_cur_queue; 147 }; 148 149 TAILQ_HEAD(umtxq_head, umtx_q); 150 151 /* Per-key wait-queue */ 152 struct umtxq_queue { 153 struct umtxq_head head; 154 struct umtx_key key; 155 LIST_ENTRY(umtxq_queue) link; 156 int length; 157 }; 158 159 LIST_HEAD(umtxq_list, umtxq_queue); 160 161 /* Userland lock object's wait-queue chain */ 162 struct umtxq_chain { 163 /* Lock for this chain. */ 164 struct mtx uc_lock; 165 166 /* List of sleep queues. */ 167 struct umtxq_list uc_queue[2]; 168 #define UMTX_SHARED_QUEUE 0 169 #define UMTX_EXCLUSIVE_QUEUE 1 170 171 LIST_HEAD(, umtxq_queue) uc_spare_queue; 172 173 /* Busy flag */ 174 char uc_busy; 175 176 /* Chain lock waiters */ 177 int uc_waiters; 178 179 /* All PI in the list */ 180 TAILQ_HEAD(,umtx_pi) uc_pi_list; 181 182 #ifdef UMTX_PROFILING 183 u_int length; 184 u_int max_length; 185 #endif 186 }; 187 188 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) 189 190 /* 191 * Don't propagate time-sharing priority, there is a security reason, 192 * a user can simply introduce PI-mutex, let thread A lock the mutex, 193 * and let another thread B block on the mutex, because B is 194 * sleeping, its priority will be boosted, this causes A's priority to 195 * be boosted via priority propagating too and will never be lowered even 196 * if it is using 100%CPU, this is unfair to other processes. 197 */ 198 199 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ 200 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ 201 PRI_MAX_TIMESHARE : (td)->td_user_pri) 202 203 #define GOLDEN_RATIO_PRIME 2654404609U 204 #ifndef UMTX_CHAINS 205 #define UMTX_CHAINS 512 206 #endif 207 #define UMTX_SHIFTS (__WORD_BIT - 9) 208 209 #define GET_SHARE(flags) \ 210 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) 211 212 #define BUSY_SPINS 200 213 214 struct abs_timeout { 215 int clockid; 216 bool is_abs_real; /* TIMER_ABSTIME && CLOCK_REALTIME* */ 217 struct timespec cur; 218 struct timespec end; 219 }; 220 221 #ifdef COMPAT_FREEBSD32 222 struct umutex32 { 223 volatile __lwpid_t m_owner; /* Owner of the mutex */ 224 __uint32_t m_flags; /* Flags of the mutex */ 225 __uint32_t m_ceilings[2]; /* Priority protect ceiling */ 226 __uint32_t m_rb_lnk; /* Robust linkage */ 227 __uint32_t m_pad; 228 __uint32_t m_spare[2]; 229 }; 230 231 _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32"); 232 _Static_assert(__offsetof(struct umutex, m_spare[0]) == 233 __offsetof(struct umutex32, m_spare[0]), "m_spare32"); 234 #endif 235 236 int umtx_shm_vnobj_persistent = 0; 237 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN, 238 &umtx_shm_vnobj_persistent, 0, 239 "False forces destruction of umtx attached to file, on last close"); 240 static int umtx_max_rb = 1000; 241 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN, 242 &umtx_max_rb, 0, 243 ""); 244 245 static uma_zone_t umtx_pi_zone; 246 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; 247 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); 248 static int umtx_pi_allocated; 249 250 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); 251 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, 252 &umtx_pi_allocated, 0, "Allocated umtx_pi"); 253 static int umtx_verbose_rb = 1; 254 SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN, 255 &umtx_verbose_rb, 0, 256 ""); 257 258 #ifdef UMTX_PROFILING 259 static long max_length; 260 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length"); 261 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD, 0, "umtx chain stats"); 262 #endif 263 264 static void abs_timeout_update(struct abs_timeout *timo); 265 266 static void umtx_shm_init(void); 267 static void umtxq_sysinit(void *); 268 static void umtxq_hash(struct umtx_key *key); 269 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); 270 static void umtxq_lock(struct umtx_key *key); 271 static void umtxq_unlock(struct umtx_key *key); 272 static void umtxq_busy(struct umtx_key *key); 273 static void umtxq_unbusy(struct umtx_key *key); 274 static void umtxq_insert_queue(struct umtx_q *uq, int q); 275 static void umtxq_remove_queue(struct umtx_q *uq, int q); 276 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *); 277 static int umtxq_count(struct umtx_key *key); 278 static struct umtx_pi *umtx_pi_alloc(int); 279 static void umtx_pi_free(struct umtx_pi *pi); 280 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, 281 bool rb); 282 static void umtx_thread_cleanup(struct thread *td); 283 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, 284 struct image_params *imgp __unused); 285 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); 286 287 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE) 288 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE) 289 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE) 290 291 static struct mtx umtx_lock; 292 293 #ifdef UMTX_PROFILING 294 static void 295 umtx_init_profiling(void) 296 { 297 struct sysctl_oid *chain_oid; 298 char chain_name[10]; 299 int i; 300 301 for (i = 0; i < UMTX_CHAINS; ++i) { 302 snprintf(chain_name, sizeof(chain_name), "%d", i); 303 chain_oid = SYSCTL_ADD_NODE(NULL, 304 SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO, 305 chain_name, CTLFLAG_RD, NULL, "umtx hash stats"); 306 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO, 307 "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL); 308 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO, 309 "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL); 310 } 311 } 312 313 static int 314 sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS) 315 { 316 char buf[512]; 317 struct sbuf sb; 318 struct umtxq_chain *uc; 319 u_int fract, i, j, tot, whole; 320 u_int sf0, sf1, sf2, sf3, sf4; 321 u_int si0, si1, si2, si3, si4; 322 u_int sw0, sw1, sw2, sw3, sw4; 323 324 sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN); 325 for (i = 0; i < 2; i++) { 326 tot = 0; 327 for (j = 0; j < UMTX_CHAINS; ++j) { 328 uc = &umtxq_chains[i][j]; 329 mtx_lock(&uc->uc_lock); 330 tot += uc->max_length; 331 mtx_unlock(&uc->uc_lock); 332 } 333 if (tot == 0) 334 sbuf_printf(&sb, "%u) Empty ", i); 335 else { 336 sf0 = sf1 = sf2 = sf3 = sf4 = 0; 337 si0 = si1 = si2 = si3 = si4 = 0; 338 sw0 = sw1 = sw2 = sw3 = sw4 = 0; 339 for (j = 0; j < UMTX_CHAINS; j++) { 340 uc = &umtxq_chains[i][j]; 341 mtx_lock(&uc->uc_lock); 342 whole = uc->max_length * 100; 343 mtx_unlock(&uc->uc_lock); 344 fract = (whole % tot) * 100; 345 if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) { 346 sf0 = fract; 347 si0 = j; 348 sw0 = whole; 349 } else if (UPROF_PERC_BIGGER(whole, fract, sw1, 350 sf1)) { 351 sf1 = fract; 352 si1 = j; 353 sw1 = whole; 354 } else if (UPROF_PERC_BIGGER(whole, fract, sw2, 355 sf2)) { 356 sf2 = fract; 357 si2 = j; 358 sw2 = whole; 359 } else if (UPROF_PERC_BIGGER(whole, fract, sw3, 360 sf3)) { 361 sf3 = fract; 362 si3 = j; 363 sw3 = whole; 364 } else if (UPROF_PERC_BIGGER(whole, fract, sw4, 365 sf4)) { 366 sf4 = fract; 367 si4 = j; 368 sw4 = whole; 369 } 370 } 371 sbuf_printf(&sb, "queue %u:\n", i); 372 sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot, 373 sf0 / tot, si0); 374 sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot, 375 sf1 / tot, si1); 376 sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot, 377 sf2 / tot, si2); 378 sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot, 379 sf3 / tot, si3); 380 sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot, 381 sf4 / tot, si4); 382 } 383 } 384 sbuf_trim(&sb); 385 sbuf_finish(&sb); 386 sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 387 sbuf_delete(&sb); 388 return (0); 389 } 390 391 static int 392 sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS) 393 { 394 struct umtxq_chain *uc; 395 u_int i, j; 396 int clear, error; 397 398 clear = 0; 399 error = sysctl_handle_int(oidp, &clear, 0, req); 400 if (error != 0 || req->newptr == NULL) 401 return (error); 402 403 if (clear != 0) { 404 for (i = 0; i < 2; ++i) { 405 for (j = 0; j < UMTX_CHAINS; ++j) { 406 uc = &umtxq_chains[i][j]; 407 mtx_lock(&uc->uc_lock); 408 uc->length = 0; 409 uc->max_length = 0; 410 mtx_unlock(&uc->uc_lock); 411 } 412 } 413 } 414 return (0); 415 } 416 417 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear, 418 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, 419 sysctl_debug_umtx_chains_clear, "I", "Clear umtx chains statistics"); 420 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks, 421 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0, 422 sysctl_debug_umtx_chains_peaks, "A", "Highest peaks in chains max length"); 423 #endif 424 425 static void 426 umtxq_sysinit(void *arg __unused) 427 { 428 int i, j; 429 430 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), 431 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 432 for (i = 0; i < 2; ++i) { 433 for (j = 0; j < UMTX_CHAINS; ++j) { 434 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL, 435 MTX_DEF | MTX_DUPOK); 436 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]); 437 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]); 438 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue); 439 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list); 440 umtxq_chains[i][j].uc_busy = 0; 441 umtxq_chains[i][j].uc_waiters = 0; 442 #ifdef UMTX_PROFILING 443 umtxq_chains[i][j].length = 0; 444 umtxq_chains[i][j].max_length = 0; 445 #endif 446 } 447 } 448 #ifdef UMTX_PROFILING 449 umtx_init_profiling(); 450 #endif 451 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_DEF); 452 EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, 453 EVENTHANDLER_PRI_ANY); 454 umtx_shm_init(); 455 } 456 457 struct umtx_q * 458 umtxq_alloc(void) 459 { 460 struct umtx_q *uq; 461 462 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); 463 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, 464 M_WAITOK | M_ZERO); 465 TAILQ_INIT(&uq->uq_spare_queue->head); 466 TAILQ_INIT(&uq->uq_pi_contested); 467 uq->uq_inherited_pri = PRI_MAX; 468 return (uq); 469 } 470 471 void 472 umtxq_free(struct umtx_q *uq) 473 { 474 475 MPASS(uq->uq_spare_queue != NULL); 476 free(uq->uq_spare_queue, M_UMTX); 477 free(uq, M_UMTX); 478 } 479 480 static inline void 481 umtxq_hash(struct umtx_key *key) 482 { 483 unsigned n; 484 485 n = (uintptr_t)key->info.both.a + key->info.both.b; 486 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; 487 } 488 489 static inline struct umtxq_chain * 490 umtxq_getchain(struct umtx_key *key) 491 { 492 493 if (key->type <= TYPE_SEM) 494 return (&umtxq_chains[1][key->hash]); 495 return (&umtxq_chains[0][key->hash]); 496 } 497 498 /* 499 * Lock a chain. 500 */ 501 static inline void 502 umtxq_lock(struct umtx_key *key) 503 { 504 struct umtxq_chain *uc; 505 506 uc = umtxq_getchain(key); 507 mtx_lock(&uc->uc_lock); 508 } 509 510 /* 511 * Unlock a chain. 512 */ 513 static inline void 514 umtxq_unlock(struct umtx_key *key) 515 { 516 struct umtxq_chain *uc; 517 518 uc = umtxq_getchain(key); 519 mtx_unlock(&uc->uc_lock); 520 } 521 522 /* 523 * Set chain to busy state when following operation 524 * may be blocked (kernel mutex can not be used). 525 */ 526 static inline void 527 umtxq_busy(struct umtx_key *key) 528 { 529 struct umtxq_chain *uc; 530 531 uc = umtxq_getchain(key); 532 mtx_assert(&uc->uc_lock, MA_OWNED); 533 if (uc->uc_busy) { 534 #ifdef SMP 535 if (smp_cpus > 1) { 536 int count = BUSY_SPINS; 537 if (count > 0) { 538 umtxq_unlock(key); 539 while (uc->uc_busy && --count > 0) 540 cpu_spinwait(); 541 umtxq_lock(key); 542 } 543 } 544 #endif 545 while (uc->uc_busy) { 546 uc->uc_waiters++; 547 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); 548 uc->uc_waiters--; 549 } 550 } 551 uc->uc_busy = 1; 552 } 553 554 /* 555 * Unbusy a chain. 556 */ 557 static inline void 558 umtxq_unbusy(struct umtx_key *key) 559 { 560 struct umtxq_chain *uc; 561 562 uc = umtxq_getchain(key); 563 mtx_assert(&uc->uc_lock, MA_OWNED); 564 KASSERT(uc->uc_busy != 0, ("not busy")); 565 uc->uc_busy = 0; 566 if (uc->uc_waiters) 567 wakeup_one(uc); 568 } 569 570 static inline void 571 umtxq_unbusy_unlocked(struct umtx_key *key) 572 { 573 574 umtxq_lock(key); 575 umtxq_unbusy(key); 576 umtxq_unlock(key); 577 } 578 579 static struct umtxq_queue * 580 umtxq_queue_lookup(struct umtx_key *key, int q) 581 { 582 struct umtxq_queue *uh; 583 struct umtxq_chain *uc; 584 585 uc = umtxq_getchain(key); 586 UMTXQ_LOCKED_ASSERT(uc); 587 LIST_FOREACH(uh, &uc->uc_queue[q], link) { 588 if (umtx_key_match(&uh->key, key)) 589 return (uh); 590 } 591 592 return (NULL); 593 } 594 595 static inline void 596 umtxq_insert_queue(struct umtx_q *uq, int q) 597 { 598 struct umtxq_queue *uh; 599 struct umtxq_chain *uc; 600 601 uc = umtxq_getchain(&uq->uq_key); 602 UMTXQ_LOCKED_ASSERT(uc); 603 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue")); 604 uh = umtxq_queue_lookup(&uq->uq_key, q); 605 if (uh != NULL) { 606 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link); 607 } else { 608 uh = uq->uq_spare_queue; 609 uh->key = uq->uq_key; 610 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link); 611 #ifdef UMTX_PROFILING 612 uc->length++; 613 if (uc->length > uc->max_length) { 614 uc->max_length = uc->length; 615 if (uc->max_length > max_length) 616 max_length = uc->max_length; 617 } 618 #endif 619 } 620 uq->uq_spare_queue = NULL; 621 622 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link); 623 uh->length++; 624 uq->uq_flags |= UQF_UMTXQ; 625 uq->uq_cur_queue = uh; 626 return; 627 } 628 629 static inline void 630 umtxq_remove_queue(struct umtx_q *uq, int q) 631 { 632 struct umtxq_chain *uc; 633 struct umtxq_queue *uh; 634 635 uc = umtxq_getchain(&uq->uq_key); 636 UMTXQ_LOCKED_ASSERT(uc); 637 if (uq->uq_flags & UQF_UMTXQ) { 638 uh = uq->uq_cur_queue; 639 TAILQ_REMOVE(&uh->head, uq, uq_link); 640 uh->length--; 641 uq->uq_flags &= ~UQF_UMTXQ; 642 if (TAILQ_EMPTY(&uh->head)) { 643 KASSERT(uh->length == 0, 644 ("inconsistent umtxq_queue length")); 645 #ifdef UMTX_PROFILING 646 uc->length--; 647 #endif 648 LIST_REMOVE(uh, link); 649 } else { 650 uh = LIST_FIRST(&uc->uc_spare_queue); 651 KASSERT(uh != NULL, ("uc_spare_queue is empty")); 652 LIST_REMOVE(uh, link); 653 } 654 uq->uq_spare_queue = uh; 655 uq->uq_cur_queue = NULL; 656 } 657 } 658 659 /* 660 * Check if there are multiple waiters 661 */ 662 static int 663 umtxq_count(struct umtx_key *key) 664 { 665 struct umtxq_queue *uh; 666 667 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key)); 668 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); 669 if (uh != NULL) 670 return (uh->length); 671 return (0); 672 } 673 674 /* 675 * Check if there are multiple PI waiters and returns first 676 * waiter. 677 */ 678 static int 679 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) 680 { 681 struct umtxq_queue *uh; 682 683 *first = NULL; 684 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key)); 685 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); 686 if (uh != NULL) { 687 *first = TAILQ_FIRST(&uh->head); 688 return (uh->length); 689 } 690 return (0); 691 } 692 693 /* 694 * Check for possible stops and suspensions while executing a umtx 695 * locking operation. 696 * 697 * The sleep argument controls whether the function can handle a stop 698 * request itself or it should return ERESTART and the request is 699 * proceed at the kernel/user boundary in ast. 700 * 701 * Typically, when retrying due to casueword(9) failure (rv == 1), we 702 * should handle the stop requests there, with exception of cases when 703 * the thread busied the umtx key, or when functions return 704 * immediately if umtxq_check_susp() returned non-zero. On the other 705 * hand, retrying the whole lock operation, we better not stop there 706 * but delegate the handling to ast. 707 * 708 * If the request is for thread termination P_SINGLE_EXIT, we cannot 709 * handle it at all, and simply return EINTR. 710 */ 711 static int 712 umtxq_check_susp(struct thread *td, bool sleep) 713 { 714 struct proc *p; 715 int error; 716 717 /* 718 * The check for TDF_NEEDSUSPCHK is racy, but it is enough to 719 * eventually break the lockstep loop. 720 */ 721 if ((td->td_flags & TDF_NEEDSUSPCHK) == 0) 722 return (0); 723 error = 0; 724 p = td->td_proc; 725 PROC_LOCK(p); 726 if (P_SHOULDSTOP(p) || 727 ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) { 728 if (p->p_flag & P_SINGLE_EXIT) 729 error = EINTR; 730 else 731 error = sleep ? thread_suspend_check(0) : ERESTART; 732 } 733 PROC_UNLOCK(p); 734 return (error); 735 } 736 737 /* 738 * Wake up threads waiting on an userland object. 739 */ 740 741 static int 742 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q) 743 { 744 struct umtxq_queue *uh; 745 struct umtx_q *uq; 746 int ret; 747 748 ret = 0; 749 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key)); 750 uh = umtxq_queue_lookup(key, q); 751 if (uh != NULL) { 752 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) { 753 umtxq_remove_queue(uq, q); 754 wakeup(uq); 755 if (++ret >= n_wake) 756 return (ret); 757 } 758 } 759 return (ret); 760 } 761 762 763 /* 764 * Wake up specified thread. 765 */ 766 static inline void 767 umtxq_signal_thread(struct umtx_q *uq) 768 { 769 770 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key)); 771 umtxq_remove(uq); 772 wakeup(uq); 773 } 774 775 static inline int 776 tstohz(const struct timespec *tsp) 777 { 778 struct timeval tv; 779 780 TIMESPEC_TO_TIMEVAL(&tv, tsp); 781 return tvtohz(&tv); 782 } 783 784 static void 785 abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute, 786 const struct timespec *timeout) 787 { 788 789 timo->clockid = clockid; 790 if (!absolute) { 791 timo->is_abs_real = false; 792 abs_timeout_update(timo); 793 timespecadd(&timo->cur, timeout, &timo->end); 794 } else { 795 timo->end = *timeout; 796 timo->is_abs_real = clockid == CLOCK_REALTIME || 797 clockid == CLOCK_REALTIME_FAST || 798 clockid == CLOCK_REALTIME_PRECISE; 799 /* 800 * If is_abs_real, umtxq_sleep will read the clock 801 * after setting td_rtcgen; otherwise, read it here. 802 */ 803 if (!timo->is_abs_real) { 804 abs_timeout_update(timo); 805 } 806 } 807 } 808 809 static void 810 abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime) 811 { 812 813 abs_timeout_init(timo, umtxtime->_clockid, 814 (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); 815 } 816 817 static inline void 818 abs_timeout_update(struct abs_timeout *timo) 819 { 820 821 kern_clock_gettime(curthread, timo->clockid, &timo->cur); 822 } 823 824 static int 825 abs_timeout_gethz(struct abs_timeout *timo) 826 { 827 struct timespec tts; 828 829 if (timespeccmp(&timo->end, &timo->cur, <=)) 830 return (-1); 831 timespecsub(&timo->end, &timo->cur, &tts); 832 return (tstohz(&tts)); 833 } 834 835 static uint32_t 836 umtx_unlock_val(uint32_t flags, bool rb) 837 { 838 839 if (rb) 840 return (UMUTEX_RB_OWNERDEAD); 841 else if ((flags & UMUTEX_NONCONSISTENT) != 0) 842 return (UMUTEX_RB_NOTRECOV); 843 else 844 return (UMUTEX_UNOWNED); 845 846 } 847 848 /* 849 * Put thread into sleep state, before sleeping, check if 850 * thread was removed from umtx queue. 851 */ 852 static inline int 853 umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime) 854 { 855 struct umtxq_chain *uc; 856 int error, timo; 857 858 if (abstime != NULL && abstime->is_abs_real) { 859 curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation); 860 abs_timeout_update(abstime); 861 } 862 863 uc = umtxq_getchain(&uq->uq_key); 864 UMTXQ_LOCKED_ASSERT(uc); 865 for (;;) { 866 if (!(uq->uq_flags & UQF_UMTXQ)) { 867 error = 0; 868 break; 869 } 870 if (abstime != NULL) { 871 timo = abs_timeout_gethz(abstime); 872 if (timo < 0) { 873 error = ETIMEDOUT; 874 break; 875 } 876 } else 877 timo = 0; 878 error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo); 879 if (error == EINTR || error == ERESTART) { 880 umtxq_lock(&uq->uq_key); 881 break; 882 } 883 if (abstime != NULL) { 884 if (abstime->is_abs_real) 885 curthread->td_rtcgen = 886 atomic_load_acq_int(&rtc_generation); 887 abs_timeout_update(abstime); 888 } 889 umtxq_lock(&uq->uq_key); 890 } 891 892 curthread->td_rtcgen = 0; 893 return (error); 894 } 895 896 /* 897 * Convert userspace address into unique logical address. 898 */ 899 int 900 umtx_key_get(const void *addr, int type, int share, struct umtx_key *key) 901 { 902 struct thread *td = curthread; 903 vm_map_t map; 904 vm_map_entry_t entry; 905 vm_pindex_t pindex; 906 vm_prot_t prot; 907 boolean_t wired; 908 909 key->type = type; 910 if (share == THREAD_SHARE) { 911 key->shared = 0; 912 key->info.private.vs = td->td_proc->p_vmspace; 913 key->info.private.addr = (uintptr_t)addr; 914 } else { 915 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE); 916 map = &td->td_proc->p_vmspace->vm_map; 917 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE, 918 &entry, &key->info.shared.object, &pindex, &prot, 919 &wired) != KERN_SUCCESS) { 920 return (EFAULT); 921 } 922 923 if ((share == PROCESS_SHARE) || 924 (share == AUTO_SHARE && 925 VM_INHERIT_SHARE == entry->inheritance)) { 926 key->shared = 1; 927 key->info.shared.offset = (vm_offset_t)addr - 928 entry->start + entry->offset; 929 vm_object_reference(key->info.shared.object); 930 } else { 931 key->shared = 0; 932 key->info.private.vs = td->td_proc->p_vmspace; 933 key->info.private.addr = (uintptr_t)addr; 934 } 935 vm_map_lookup_done(map, entry); 936 } 937 938 umtxq_hash(key); 939 return (0); 940 } 941 942 /* 943 * Release key. 944 */ 945 void 946 umtx_key_release(struct umtx_key *key) 947 { 948 if (key->shared) 949 vm_object_deallocate(key->info.shared.object); 950 } 951 952 /* 953 * Fetch and compare value, sleep on the address if value is not changed. 954 */ 955 static int 956 do_wait(struct thread *td, void *addr, u_long id, 957 struct _umtx_time *timeout, int compat32, int is_private) 958 { 959 struct abs_timeout timo; 960 struct umtx_q *uq; 961 u_long tmp; 962 uint32_t tmp32; 963 int error = 0; 964 965 uq = td->td_umtxq; 966 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, 967 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) 968 return (error); 969 970 if (timeout != NULL) 971 abs_timeout_init2(&timo, timeout); 972 973 umtxq_lock(&uq->uq_key); 974 umtxq_insert(uq); 975 umtxq_unlock(&uq->uq_key); 976 if (compat32 == 0) { 977 error = fueword(addr, &tmp); 978 if (error != 0) 979 error = EFAULT; 980 } else { 981 error = fueword32(addr, &tmp32); 982 if (error == 0) 983 tmp = tmp32; 984 else 985 error = EFAULT; 986 } 987 umtxq_lock(&uq->uq_key); 988 if (error == 0) { 989 if (tmp == id) 990 error = umtxq_sleep(uq, "uwait", timeout == NULL ? 991 NULL : &timo); 992 if ((uq->uq_flags & UQF_UMTXQ) == 0) 993 error = 0; 994 else 995 umtxq_remove(uq); 996 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) { 997 umtxq_remove(uq); 998 } 999 umtxq_unlock(&uq->uq_key); 1000 umtx_key_release(&uq->uq_key); 1001 if (error == ERESTART) 1002 error = EINTR; 1003 return (error); 1004 } 1005 1006 /* 1007 * Wake up threads sleeping on the specified address. 1008 */ 1009 int 1010 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) 1011 { 1012 struct umtx_key key; 1013 int ret; 1014 1015 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, 1016 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) 1017 return (ret); 1018 umtxq_lock(&key); 1019 umtxq_signal(&key, n_wake); 1020 umtxq_unlock(&key); 1021 umtx_key_release(&key); 1022 return (0); 1023 } 1024 1025 /* 1026 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1027 */ 1028 static int 1029 do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, 1030 struct _umtx_time *timeout, int mode) 1031 { 1032 struct abs_timeout timo; 1033 struct umtx_q *uq; 1034 uint32_t owner, old, id; 1035 int error, rv; 1036 1037 id = td->td_tid; 1038 uq = td->td_umtxq; 1039 error = 0; 1040 if (timeout != NULL) 1041 abs_timeout_init2(&timo, timeout); 1042 1043 /* 1044 * Care must be exercised when dealing with umtx structure. It 1045 * can fault on any access. 1046 */ 1047 for (;;) { 1048 rv = fueword32(&m->m_owner, &owner); 1049 if (rv == -1) 1050 return (EFAULT); 1051 if (mode == _UMUTEX_WAIT) { 1052 if (owner == UMUTEX_UNOWNED || 1053 owner == UMUTEX_CONTESTED || 1054 owner == UMUTEX_RB_OWNERDEAD || 1055 owner == UMUTEX_RB_NOTRECOV) 1056 return (0); 1057 } else { 1058 /* 1059 * Robust mutex terminated. Kernel duty is to 1060 * return EOWNERDEAD to the userspace. The 1061 * umutex.m_flags UMUTEX_NONCONSISTENT is set 1062 * by the common userspace code. 1063 */ 1064 if (owner == UMUTEX_RB_OWNERDEAD) { 1065 rv = casueword32(&m->m_owner, 1066 UMUTEX_RB_OWNERDEAD, &owner, 1067 id | UMUTEX_CONTESTED); 1068 if (rv == -1) 1069 return (EFAULT); 1070 if (rv == 0) { 1071 MPASS(owner == UMUTEX_RB_OWNERDEAD); 1072 return (EOWNERDEAD); /* success */ 1073 } 1074 MPASS(rv == 1); 1075 rv = umtxq_check_susp(td, false); 1076 if (rv != 0) 1077 return (rv); 1078 continue; 1079 } 1080 if (owner == UMUTEX_RB_NOTRECOV) 1081 return (ENOTRECOVERABLE); 1082 1083 /* 1084 * Try the uncontested case. This should be 1085 * done in userland. 1086 */ 1087 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, 1088 &owner, id); 1089 /* The address was invalid. */ 1090 if (rv == -1) 1091 return (EFAULT); 1092 1093 /* The acquire succeeded. */ 1094 if (rv == 0) { 1095 MPASS(owner == UMUTEX_UNOWNED); 1096 return (0); 1097 } 1098 1099 /* 1100 * If no one owns it but it is contested try 1101 * to acquire it. 1102 */ 1103 MPASS(rv == 1); 1104 if (owner == UMUTEX_CONTESTED) { 1105 rv = casueword32(&m->m_owner, 1106 UMUTEX_CONTESTED, &owner, 1107 id | UMUTEX_CONTESTED); 1108 /* The address was invalid. */ 1109 if (rv == -1) 1110 return (EFAULT); 1111 if (rv == 0) { 1112 MPASS(owner == UMUTEX_CONTESTED); 1113 return (0); 1114 } 1115 if (rv == 1) { 1116 rv = umtxq_check_susp(td, false); 1117 if (rv != 0) 1118 return (rv); 1119 } 1120 1121 /* 1122 * If this failed the lock has 1123 * changed, restart. 1124 */ 1125 continue; 1126 } 1127 1128 /* rv == 1 but not contested, likely store failure */ 1129 rv = umtxq_check_susp(td, false); 1130 if (rv != 0) 1131 return (rv); 1132 } 1133 1134 if (mode == _UMUTEX_TRY) 1135 return (EBUSY); 1136 1137 /* 1138 * If we caught a signal, we have retried and now 1139 * exit immediately. 1140 */ 1141 if (error != 0) 1142 return (error); 1143 1144 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, 1145 GET_SHARE(flags), &uq->uq_key)) != 0) 1146 return (error); 1147 1148 umtxq_lock(&uq->uq_key); 1149 umtxq_busy(&uq->uq_key); 1150 umtxq_insert(uq); 1151 umtxq_unlock(&uq->uq_key); 1152 1153 /* 1154 * Set the contested bit so that a release in user space 1155 * knows to use the system call for unlock. If this fails 1156 * either some one else has acquired the lock or it has been 1157 * released. 1158 */ 1159 rv = casueword32(&m->m_owner, owner, &old, 1160 owner | UMUTEX_CONTESTED); 1161 1162 /* The address was invalid or casueword failed to store. */ 1163 if (rv == -1 || rv == 1) { 1164 umtxq_lock(&uq->uq_key); 1165 umtxq_remove(uq); 1166 umtxq_unbusy(&uq->uq_key); 1167 umtxq_unlock(&uq->uq_key); 1168 umtx_key_release(&uq->uq_key); 1169 if (rv == -1) 1170 return (EFAULT); 1171 if (rv == 1) { 1172 rv = umtxq_check_susp(td, false); 1173 if (rv != 0) 1174 return (rv); 1175 } 1176 continue; 1177 } 1178 1179 /* 1180 * We set the contested bit, sleep. Otherwise the lock changed 1181 * and we need to retry or we lost a race to the thread 1182 * unlocking the umtx. 1183 */ 1184 umtxq_lock(&uq->uq_key); 1185 umtxq_unbusy(&uq->uq_key); 1186 MPASS(old == owner); 1187 error = umtxq_sleep(uq, "umtxn", timeout == NULL ? 1188 NULL : &timo); 1189 umtxq_remove(uq); 1190 umtxq_unlock(&uq->uq_key); 1191 umtx_key_release(&uq->uq_key); 1192 1193 if (error == 0) 1194 error = umtxq_check_susp(td, false); 1195 } 1196 1197 return (0); 1198 } 1199 1200 /* 1201 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. 1202 */ 1203 static int 1204 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb) 1205 { 1206 struct umtx_key key; 1207 uint32_t owner, old, id, newlock; 1208 int error, count; 1209 1210 id = td->td_tid; 1211 1212 again: 1213 /* 1214 * Make sure we own this mtx. 1215 */ 1216 error = fueword32(&m->m_owner, &owner); 1217 if (error == -1) 1218 return (EFAULT); 1219 1220 if ((owner & ~UMUTEX_CONTESTED) != id) 1221 return (EPERM); 1222 1223 newlock = umtx_unlock_val(flags, rb); 1224 if ((owner & UMUTEX_CONTESTED) == 0) { 1225 error = casueword32(&m->m_owner, owner, &old, newlock); 1226 if (error == -1) 1227 return (EFAULT); 1228 if (error == 1) { 1229 error = umtxq_check_susp(td, false); 1230 if (error != 0) 1231 return (error); 1232 goto again; 1233 } 1234 MPASS(old == owner); 1235 return (0); 1236 } 1237 1238 /* We should only ever be in here for contested locks */ 1239 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1240 &key)) != 0) 1241 return (error); 1242 1243 umtxq_lock(&key); 1244 umtxq_busy(&key); 1245 count = umtxq_count(&key); 1246 umtxq_unlock(&key); 1247 1248 /* 1249 * When unlocking the umtx, it must be marked as unowned if 1250 * there is zero or one thread only waiting for it. 1251 * Otherwise, it must be marked as contested. 1252 */ 1253 if (count > 1) 1254 newlock |= UMUTEX_CONTESTED; 1255 error = casueword32(&m->m_owner, owner, &old, newlock); 1256 umtxq_lock(&key); 1257 umtxq_signal(&key, 1); 1258 umtxq_unbusy(&key); 1259 umtxq_unlock(&key); 1260 umtx_key_release(&key); 1261 if (error == -1) 1262 return (EFAULT); 1263 if (error == 1) { 1264 if (old != owner) 1265 return (EINVAL); 1266 error = umtxq_check_susp(td, false); 1267 if (error != 0) 1268 return (error); 1269 goto again; 1270 } 1271 return (0); 1272 } 1273 1274 /* 1275 * Check if the mutex is available and wake up a waiter, 1276 * only for simple mutex. 1277 */ 1278 static int 1279 do_wake_umutex(struct thread *td, struct umutex *m) 1280 { 1281 struct umtx_key key; 1282 uint32_t owner; 1283 uint32_t flags; 1284 int error; 1285 int count; 1286 1287 again: 1288 error = fueword32(&m->m_owner, &owner); 1289 if (error == -1) 1290 return (EFAULT); 1291 1292 if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD && 1293 owner != UMUTEX_RB_NOTRECOV) 1294 return (0); 1295 1296 error = fueword32(&m->m_flags, &flags); 1297 if (error == -1) 1298 return (EFAULT); 1299 1300 /* We should only ever be in here for contested locks */ 1301 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1302 &key)) != 0) 1303 return (error); 1304 1305 umtxq_lock(&key); 1306 umtxq_busy(&key); 1307 count = umtxq_count(&key); 1308 umtxq_unlock(&key); 1309 1310 if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD && 1311 owner != UMUTEX_RB_NOTRECOV) { 1312 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, 1313 UMUTEX_UNOWNED); 1314 if (error == -1) { 1315 error = EFAULT; 1316 } else if (error == 1) { 1317 umtxq_lock(&key); 1318 umtxq_unbusy(&key); 1319 umtxq_unlock(&key); 1320 umtx_key_release(&key); 1321 error = umtxq_check_susp(td, false); 1322 if (error != 0) 1323 return (error); 1324 goto again; 1325 } 1326 } 1327 1328 umtxq_lock(&key); 1329 if (error == 0 && count != 0) { 1330 MPASS((owner & ~UMUTEX_CONTESTED) == 0 || 1331 owner == UMUTEX_RB_OWNERDEAD || 1332 owner == UMUTEX_RB_NOTRECOV); 1333 umtxq_signal(&key, 1); 1334 } 1335 umtxq_unbusy(&key); 1336 umtxq_unlock(&key); 1337 umtx_key_release(&key); 1338 return (error); 1339 } 1340 1341 /* 1342 * Check if the mutex has waiters and tries to fix contention bit. 1343 */ 1344 static int 1345 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) 1346 { 1347 struct umtx_key key; 1348 uint32_t owner, old; 1349 int type; 1350 int error; 1351 int count; 1352 1353 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT | 1354 UMUTEX_ROBUST)) { 1355 case 0: 1356 case UMUTEX_ROBUST: 1357 type = TYPE_NORMAL_UMUTEX; 1358 break; 1359 case UMUTEX_PRIO_INHERIT: 1360 type = TYPE_PI_UMUTEX; 1361 break; 1362 case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST): 1363 type = TYPE_PI_ROBUST_UMUTEX; 1364 break; 1365 case UMUTEX_PRIO_PROTECT: 1366 type = TYPE_PP_UMUTEX; 1367 break; 1368 case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST): 1369 type = TYPE_PP_ROBUST_UMUTEX; 1370 break; 1371 default: 1372 return (EINVAL); 1373 } 1374 if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0) 1375 return (error); 1376 1377 owner = 0; 1378 umtxq_lock(&key); 1379 umtxq_busy(&key); 1380 count = umtxq_count(&key); 1381 umtxq_unlock(&key); 1382 1383 error = fueword32(&m->m_owner, &owner); 1384 if (error == -1) 1385 error = EFAULT; 1386 1387 /* 1388 * Only repair contention bit if there is a waiter, this means 1389 * the mutex is still being referenced by userland code, 1390 * otherwise don't update any memory. 1391 */ 1392 while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 && 1393 (count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) { 1394 error = casueword32(&m->m_owner, owner, &old, 1395 owner | UMUTEX_CONTESTED); 1396 if (error == -1) { 1397 error = EFAULT; 1398 break; 1399 } 1400 if (error == 0) { 1401 MPASS(old == owner); 1402 break; 1403 } 1404 owner = old; 1405 error = umtxq_check_susp(td, false); 1406 } 1407 1408 umtxq_lock(&key); 1409 if (error == EFAULT) { 1410 umtxq_signal(&key, INT_MAX); 1411 } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || 1412 owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) 1413 umtxq_signal(&key, 1); 1414 umtxq_unbusy(&key); 1415 umtxq_unlock(&key); 1416 umtx_key_release(&key); 1417 return (error); 1418 } 1419 1420 static inline struct umtx_pi * 1421 umtx_pi_alloc(int flags) 1422 { 1423 struct umtx_pi *pi; 1424 1425 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags); 1426 TAILQ_INIT(&pi->pi_blocked); 1427 atomic_add_int(&umtx_pi_allocated, 1); 1428 return (pi); 1429 } 1430 1431 static inline void 1432 umtx_pi_free(struct umtx_pi *pi) 1433 { 1434 uma_zfree(umtx_pi_zone, pi); 1435 atomic_add_int(&umtx_pi_allocated, -1); 1436 } 1437 1438 /* 1439 * Adjust the thread's position on a pi_state after its priority has been 1440 * changed. 1441 */ 1442 static int 1443 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) 1444 { 1445 struct umtx_q *uq, *uq1, *uq2; 1446 struct thread *td1; 1447 1448 mtx_assert(&umtx_lock, MA_OWNED); 1449 if (pi == NULL) 1450 return (0); 1451 1452 uq = td->td_umtxq; 1453 1454 /* 1455 * Check if the thread needs to be moved on the blocked chain. 1456 * It needs to be moved if either its priority is lower than 1457 * the previous thread or higher than the next thread. 1458 */ 1459 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq); 1460 uq2 = TAILQ_NEXT(uq, uq_lockq); 1461 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || 1462 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { 1463 /* 1464 * Remove thread from blocked chain and determine where 1465 * it should be moved to. 1466 */ 1467 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1468 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1469 td1 = uq1->uq_thread; 1470 MPASS(td1->td_proc->p_magic == P_MAGIC); 1471 if (UPRI(td1) > UPRI(td)) 1472 break; 1473 } 1474 1475 if (uq1 == NULL) 1476 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1477 else 1478 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1479 } 1480 return (1); 1481 } 1482 1483 static struct umtx_pi * 1484 umtx_pi_next(struct umtx_pi *pi) 1485 { 1486 struct umtx_q *uq_owner; 1487 1488 if (pi->pi_owner == NULL) 1489 return (NULL); 1490 uq_owner = pi->pi_owner->td_umtxq; 1491 if (uq_owner == NULL) 1492 return (NULL); 1493 return (uq_owner->uq_pi_blocked); 1494 } 1495 1496 /* 1497 * Floyd's Cycle-Finding Algorithm. 1498 */ 1499 static bool 1500 umtx_pi_check_loop(struct umtx_pi *pi) 1501 { 1502 struct umtx_pi *pi1; /* fast iterator */ 1503 1504 mtx_assert(&umtx_lock, MA_OWNED); 1505 if (pi == NULL) 1506 return (false); 1507 pi1 = pi; 1508 for (;;) { 1509 pi = umtx_pi_next(pi); 1510 if (pi == NULL) 1511 break; 1512 pi1 = umtx_pi_next(pi1); 1513 if (pi1 == NULL) 1514 break; 1515 pi1 = umtx_pi_next(pi1); 1516 if (pi1 == NULL) 1517 break; 1518 if (pi == pi1) 1519 return (true); 1520 } 1521 return (false); 1522 } 1523 1524 /* 1525 * Propagate priority when a thread is blocked on POSIX 1526 * PI mutex. 1527 */ 1528 static void 1529 umtx_propagate_priority(struct thread *td) 1530 { 1531 struct umtx_q *uq; 1532 struct umtx_pi *pi; 1533 int pri; 1534 1535 mtx_assert(&umtx_lock, MA_OWNED); 1536 pri = UPRI(td); 1537 uq = td->td_umtxq; 1538 pi = uq->uq_pi_blocked; 1539 if (pi == NULL) 1540 return; 1541 if (umtx_pi_check_loop(pi)) 1542 return; 1543 1544 for (;;) { 1545 td = pi->pi_owner; 1546 if (td == NULL || td == curthread) 1547 return; 1548 1549 MPASS(td->td_proc != NULL); 1550 MPASS(td->td_proc->p_magic == P_MAGIC); 1551 1552 thread_lock(td); 1553 if (td->td_lend_user_pri > pri) 1554 sched_lend_user_prio(td, pri); 1555 else { 1556 thread_unlock(td); 1557 break; 1558 } 1559 thread_unlock(td); 1560 1561 /* 1562 * Pick up the lock that td is blocked on. 1563 */ 1564 uq = td->td_umtxq; 1565 pi = uq->uq_pi_blocked; 1566 if (pi == NULL) 1567 break; 1568 /* Resort td on the list if needed. */ 1569 umtx_pi_adjust_thread(pi, td); 1570 } 1571 } 1572 1573 /* 1574 * Unpropagate priority for a PI mutex when a thread blocked on 1575 * it is interrupted by signal or resumed by others. 1576 */ 1577 static void 1578 umtx_repropagate_priority(struct umtx_pi *pi) 1579 { 1580 struct umtx_q *uq, *uq_owner; 1581 struct umtx_pi *pi2; 1582 int pri; 1583 1584 mtx_assert(&umtx_lock, MA_OWNED); 1585 1586 if (umtx_pi_check_loop(pi)) 1587 return; 1588 while (pi != NULL && pi->pi_owner != NULL) { 1589 pri = PRI_MAX; 1590 uq_owner = pi->pi_owner->td_umtxq; 1591 1592 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { 1593 uq = TAILQ_FIRST(&pi2->pi_blocked); 1594 if (uq != NULL) { 1595 if (pri > UPRI(uq->uq_thread)) 1596 pri = UPRI(uq->uq_thread); 1597 } 1598 } 1599 1600 if (pri > uq_owner->uq_inherited_pri) 1601 pri = uq_owner->uq_inherited_pri; 1602 thread_lock(pi->pi_owner); 1603 sched_lend_user_prio(pi->pi_owner, pri); 1604 thread_unlock(pi->pi_owner); 1605 if ((pi = uq_owner->uq_pi_blocked) != NULL) 1606 umtx_pi_adjust_thread(pi, uq_owner->uq_thread); 1607 } 1608 } 1609 1610 /* 1611 * Insert a PI mutex into owned list. 1612 */ 1613 static void 1614 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) 1615 { 1616 struct umtx_q *uq_owner; 1617 1618 uq_owner = owner->td_umtxq; 1619 mtx_assert(&umtx_lock, MA_OWNED); 1620 MPASS(pi->pi_owner == NULL); 1621 pi->pi_owner = owner; 1622 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); 1623 } 1624 1625 1626 /* 1627 * Disown a PI mutex, and remove it from the owned list. 1628 */ 1629 static void 1630 umtx_pi_disown(struct umtx_pi *pi) 1631 { 1632 1633 mtx_assert(&umtx_lock, MA_OWNED); 1634 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link); 1635 pi->pi_owner = NULL; 1636 } 1637 1638 /* 1639 * Claim ownership of a PI mutex. 1640 */ 1641 static int 1642 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) 1643 { 1644 struct umtx_q *uq; 1645 int pri; 1646 1647 mtx_lock(&umtx_lock); 1648 if (pi->pi_owner == owner) { 1649 mtx_unlock(&umtx_lock); 1650 return (0); 1651 } 1652 1653 if (pi->pi_owner != NULL) { 1654 /* 1655 * userland may have already messed the mutex, sigh. 1656 */ 1657 mtx_unlock(&umtx_lock); 1658 return (EPERM); 1659 } 1660 umtx_pi_setowner(pi, owner); 1661 uq = TAILQ_FIRST(&pi->pi_blocked); 1662 if (uq != NULL) { 1663 pri = UPRI(uq->uq_thread); 1664 thread_lock(owner); 1665 if (pri < UPRI(owner)) 1666 sched_lend_user_prio(owner, pri); 1667 thread_unlock(owner); 1668 } 1669 mtx_unlock(&umtx_lock); 1670 return (0); 1671 } 1672 1673 /* 1674 * Adjust a thread's order position in its blocked PI mutex, 1675 * this may result new priority propagating process. 1676 */ 1677 void 1678 umtx_pi_adjust(struct thread *td, u_char oldpri) 1679 { 1680 struct umtx_q *uq; 1681 struct umtx_pi *pi; 1682 1683 uq = td->td_umtxq; 1684 mtx_lock(&umtx_lock); 1685 /* 1686 * Pick up the lock that td is blocked on. 1687 */ 1688 pi = uq->uq_pi_blocked; 1689 if (pi != NULL) { 1690 umtx_pi_adjust_thread(pi, td); 1691 umtx_repropagate_priority(pi); 1692 } 1693 mtx_unlock(&umtx_lock); 1694 } 1695 1696 /* 1697 * Sleep on a PI mutex. 1698 */ 1699 static int 1700 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner, 1701 const char *wmesg, struct abs_timeout *timo, bool shared) 1702 { 1703 struct thread *td, *td1; 1704 struct umtx_q *uq1; 1705 int error, pri; 1706 #ifdef INVARIANTS 1707 struct umtxq_chain *uc; 1708 1709 uc = umtxq_getchain(&pi->pi_key); 1710 #endif 1711 error = 0; 1712 td = uq->uq_thread; 1713 KASSERT(td == curthread, ("inconsistent uq_thread")); 1714 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key)); 1715 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); 1716 umtxq_insert(uq); 1717 mtx_lock(&umtx_lock); 1718 if (pi->pi_owner == NULL) { 1719 mtx_unlock(&umtx_lock); 1720 td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid); 1721 mtx_lock(&umtx_lock); 1722 if (td1 != NULL) { 1723 if (pi->pi_owner == NULL) 1724 umtx_pi_setowner(pi, td1); 1725 PROC_UNLOCK(td1->td_proc); 1726 } 1727 } 1728 1729 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1730 pri = UPRI(uq1->uq_thread); 1731 if (pri > UPRI(td)) 1732 break; 1733 } 1734 1735 if (uq1 != NULL) 1736 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1737 else 1738 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1739 1740 uq->uq_pi_blocked = pi; 1741 thread_lock(td); 1742 td->td_flags |= TDF_UPIBLOCKED; 1743 thread_unlock(td); 1744 umtx_propagate_priority(td); 1745 mtx_unlock(&umtx_lock); 1746 umtxq_unbusy(&uq->uq_key); 1747 1748 error = umtxq_sleep(uq, wmesg, timo); 1749 umtxq_remove(uq); 1750 1751 mtx_lock(&umtx_lock); 1752 uq->uq_pi_blocked = NULL; 1753 thread_lock(td); 1754 td->td_flags &= ~TDF_UPIBLOCKED; 1755 thread_unlock(td); 1756 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1757 umtx_repropagate_priority(pi); 1758 mtx_unlock(&umtx_lock); 1759 umtxq_unlock(&uq->uq_key); 1760 1761 return (error); 1762 } 1763 1764 /* 1765 * Add reference count for a PI mutex. 1766 */ 1767 static void 1768 umtx_pi_ref(struct umtx_pi *pi) 1769 { 1770 1771 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key)); 1772 pi->pi_refcount++; 1773 } 1774 1775 /* 1776 * Decrease reference count for a PI mutex, if the counter 1777 * is decreased to zero, its memory space is freed. 1778 */ 1779 static void 1780 umtx_pi_unref(struct umtx_pi *pi) 1781 { 1782 struct umtxq_chain *uc; 1783 1784 uc = umtxq_getchain(&pi->pi_key); 1785 UMTXQ_LOCKED_ASSERT(uc); 1786 KASSERT(pi->pi_refcount > 0, ("invalid reference count")); 1787 if (--pi->pi_refcount == 0) { 1788 mtx_lock(&umtx_lock); 1789 if (pi->pi_owner != NULL) 1790 umtx_pi_disown(pi); 1791 KASSERT(TAILQ_EMPTY(&pi->pi_blocked), 1792 ("blocked queue not empty")); 1793 mtx_unlock(&umtx_lock); 1794 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); 1795 umtx_pi_free(pi); 1796 } 1797 } 1798 1799 /* 1800 * Find a PI mutex in hash table. 1801 */ 1802 static struct umtx_pi * 1803 umtx_pi_lookup(struct umtx_key *key) 1804 { 1805 struct umtxq_chain *uc; 1806 struct umtx_pi *pi; 1807 1808 uc = umtxq_getchain(key); 1809 UMTXQ_LOCKED_ASSERT(uc); 1810 1811 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { 1812 if (umtx_key_match(&pi->pi_key, key)) { 1813 return (pi); 1814 } 1815 } 1816 return (NULL); 1817 } 1818 1819 /* 1820 * Insert a PI mutex into hash table. 1821 */ 1822 static inline void 1823 umtx_pi_insert(struct umtx_pi *pi) 1824 { 1825 struct umtxq_chain *uc; 1826 1827 uc = umtxq_getchain(&pi->pi_key); 1828 UMTXQ_LOCKED_ASSERT(uc); 1829 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); 1830 } 1831 1832 /* 1833 * Lock a PI mutex. 1834 */ 1835 static int 1836 do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, 1837 struct _umtx_time *timeout, int try) 1838 { 1839 struct abs_timeout timo; 1840 struct umtx_q *uq; 1841 struct umtx_pi *pi, *new_pi; 1842 uint32_t id, old_owner, owner, old; 1843 int error, rv; 1844 1845 id = td->td_tid; 1846 uq = td->td_umtxq; 1847 1848 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 1849 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), 1850 &uq->uq_key)) != 0) 1851 return (error); 1852 1853 if (timeout != NULL) 1854 abs_timeout_init2(&timo, timeout); 1855 1856 umtxq_lock(&uq->uq_key); 1857 pi = umtx_pi_lookup(&uq->uq_key); 1858 if (pi == NULL) { 1859 new_pi = umtx_pi_alloc(M_NOWAIT); 1860 if (new_pi == NULL) { 1861 umtxq_unlock(&uq->uq_key); 1862 new_pi = umtx_pi_alloc(M_WAITOK); 1863 umtxq_lock(&uq->uq_key); 1864 pi = umtx_pi_lookup(&uq->uq_key); 1865 if (pi != NULL) { 1866 umtx_pi_free(new_pi); 1867 new_pi = NULL; 1868 } 1869 } 1870 if (new_pi != NULL) { 1871 new_pi->pi_key = uq->uq_key; 1872 umtx_pi_insert(new_pi); 1873 pi = new_pi; 1874 } 1875 } 1876 umtx_pi_ref(pi); 1877 umtxq_unlock(&uq->uq_key); 1878 1879 /* 1880 * Care must be exercised when dealing with umtx structure. It 1881 * can fault on any access. 1882 */ 1883 for (;;) { 1884 /* 1885 * Try the uncontested case. This should be done in userland. 1886 */ 1887 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); 1888 /* The address was invalid. */ 1889 if (rv == -1) { 1890 error = EFAULT; 1891 break; 1892 } 1893 /* The acquire succeeded. */ 1894 if (rv == 0) { 1895 MPASS(owner == UMUTEX_UNOWNED); 1896 error = 0; 1897 break; 1898 } 1899 1900 if (owner == UMUTEX_RB_NOTRECOV) { 1901 error = ENOTRECOVERABLE; 1902 break; 1903 } 1904 1905 /* 1906 * Avoid overwriting a possible error from sleep due 1907 * to the pending signal with suspension check result. 1908 */ 1909 if (error == 0) { 1910 error = umtxq_check_susp(td, true); 1911 if (error != 0) 1912 break; 1913 } 1914 1915 /* If no one owns it but it is contested try to acquire it. */ 1916 if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { 1917 old_owner = owner; 1918 rv = casueword32(&m->m_owner, owner, &owner, 1919 id | UMUTEX_CONTESTED); 1920 /* The address was invalid. */ 1921 if (rv == -1) { 1922 error = EFAULT; 1923 break; 1924 } 1925 if (rv == 1) { 1926 if (error == 0) { 1927 error = umtxq_check_susp(td, true); 1928 if (error != 0) 1929 break; 1930 } 1931 1932 /* 1933 * If this failed the lock could 1934 * changed, restart. 1935 */ 1936 continue; 1937 } 1938 1939 MPASS(rv == 0); 1940 MPASS(owner == old_owner); 1941 umtxq_lock(&uq->uq_key); 1942 umtxq_busy(&uq->uq_key); 1943 error = umtx_pi_claim(pi, td); 1944 umtxq_unbusy(&uq->uq_key); 1945 umtxq_unlock(&uq->uq_key); 1946 if (error != 0) { 1947 /* 1948 * Since we're going to return an 1949 * error, restore the m_owner to its 1950 * previous, unowned state to avoid 1951 * compounding the problem. 1952 */ 1953 (void)casuword32(&m->m_owner, 1954 id | UMUTEX_CONTESTED, old_owner); 1955 } 1956 if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD) 1957 error = EOWNERDEAD; 1958 break; 1959 } 1960 1961 if ((owner & ~UMUTEX_CONTESTED) == id) { 1962 error = EDEADLK; 1963 break; 1964 } 1965 1966 if (try != 0) { 1967 error = EBUSY; 1968 break; 1969 } 1970 1971 /* 1972 * If we caught a signal, we have retried and now 1973 * exit immediately. 1974 */ 1975 if (error != 0) 1976 break; 1977 1978 umtxq_lock(&uq->uq_key); 1979 umtxq_busy(&uq->uq_key); 1980 umtxq_unlock(&uq->uq_key); 1981 1982 /* 1983 * Set the contested bit so that a release in user space 1984 * knows to use the system call for unlock. If this fails 1985 * either some one else has acquired the lock or it has been 1986 * released. 1987 */ 1988 rv = casueword32(&m->m_owner, owner, &old, owner | 1989 UMUTEX_CONTESTED); 1990 1991 /* The address was invalid. */ 1992 if (rv == -1) { 1993 umtxq_unbusy_unlocked(&uq->uq_key); 1994 error = EFAULT; 1995 break; 1996 } 1997 if (rv == 1) { 1998 umtxq_unbusy_unlocked(&uq->uq_key); 1999 error = umtxq_check_susp(td, true); 2000 if (error != 0) 2001 break; 2002 2003 /* 2004 * The lock changed and we need to retry or we 2005 * lost a race to the thread unlocking the 2006 * umtx. Note that the UMUTEX_RB_OWNERDEAD 2007 * value for owner is impossible there. 2008 */ 2009 continue; 2010 } 2011 2012 umtxq_lock(&uq->uq_key); 2013 2014 /* We set the contested bit, sleep. */ 2015 MPASS(old == owner); 2016 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, 2017 "umtxpi", timeout == NULL ? NULL : &timo, 2018 (flags & USYNC_PROCESS_SHARED) != 0); 2019 if (error != 0) 2020 continue; 2021 2022 error = umtxq_check_susp(td, false); 2023 if (error != 0) 2024 break; 2025 } 2026 2027 umtxq_lock(&uq->uq_key); 2028 umtx_pi_unref(pi); 2029 umtxq_unlock(&uq->uq_key); 2030 2031 umtx_key_release(&uq->uq_key); 2032 return (error); 2033 } 2034 2035 /* 2036 * Unlock a PI mutex. 2037 */ 2038 static int 2039 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb) 2040 { 2041 struct umtx_key key; 2042 struct umtx_q *uq_first, *uq_first2, *uq_me; 2043 struct umtx_pi *pi, *pi2; 2044 uint32_t id, new_owner, old, owner; 2045 int count, error, pri; 2046 2047 id = td->td_tid; 2048 2049 usrloop: 2050 /* 2051 * Make sure we own this mtx. 2052 */ 2053 error = fueword32(&m->m_owner, &owner); 2054 if (error == -1) 2055 return (EFAULT); 2056 2057 if ((owner & ~UMUTEX_CONTESTED) != id) 2058 return (EPERM); 2059 2060 new_owner = umtx_unlock_val(flags, rb); 2061 2062 /* This should be done in userland */ 2063 if ((owner & UMUTEX_CONTESTED) == 0) { 2064 error = casueword32(&m->m_owner, owner, &old, new_owner); 2065 if (error == -1) 2066 return (EFAULT); 2067 if (error == 1) { 2068 error = umtxq_check_susp(td, true); 2069 if (error != 0) 2070 return (error); 2071 goto usrloop; 2072 } 2073 if (old == owner) 2074 return (0); 2075 owner = old; 2076 } 2077 2078 /* We should only ever be in here for contested locks */ 2079 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2080 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), 2081 &key)) != 0) 2082 return (error); 2083 2084 umtxq_lock(&key); 2085 umtxq_busy(&key); 2086 count = umtxq_count_pi(&key, &uq_first); 2087 if (uq_first != NULL) { 2088 mtx_lock(&umtx_lock); 2089 pi = uq_first->uq_pi_blocked; 2090 KASSERT(pi != NULL, ("pi == NULL?")); 2091 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { 2092 mtx_unlock(&umtx_lock); 2093 umtxq_unbusy(&key); 2094 umtxq_unlock(&key); 2095 umtx_key_release(&key); 2096 /* userland messed the mutex */ 2097 return (EPERM); 2098 } 2099 uq_me = td->td_umtxq; 2100 if (pi->pi_owner == td) 2101 umtx_pi_disown(pi); 2102 /* get highest priority thread which is still sleeping. */ 2103 uq_first = TAILQ_FIRST(&pi->pi_blocked); 2104 while (uq_first != NULL && 2105 (uq_first->uq_flags & UQF_UMTXQ) == 0) { 2106 uq_first = TAILQ_NEXT(uq_first, uq_lockq); 2107 } 2108 pri = PRI_MAX; 2109 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { 2110 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); 2111 if (uq_first2 != NULL) { 2112 if (pri > UPRI(uq_first2->uq_thread)) 2113 pri = UPRI(uq_first2->uq_thread); 2114 } 2115 } 2116 thread_lock(td); 2117 sched_lend_user_prio(td, pri); 2118 thread_unlock(td); 2119 mtx_unlock(&umtx_lock); 2120 if (uq_first) 2121 umtxq_signal_thread(uq_first); 2122 } else { 2123 pi = umtx_pi_lookup(&key); 2124 /* 2125 * A umtx_pi can exist if a signal or timeout removed the 2126 * last waiter from the umtxq, but there is still 2127 * a thread in do_lock_pi() holding the umtx_pi. 2128 */ 2129 if (pi != NULL) { 2130 /* 2131 * The umtx_pi can be unowned, such as when a thread 2132 * has just entered do_lock_pi(), allocated the 2133 * umtx_pi, and unlocked the umtxq. 2134 * If the current thread owns it, it must disown it. 2135 */ 2136 mtx_lock(&umtx_lock); 2137 if (pi->pi_owner == td) 2138 umtx_pi_disown(pi); 2139 mtx_unlock(&umtx_lock); 2140 } 2141 } 2142 umtxq_unlock(&key); 2143 2144 /* 2145 * When unlocking the umtx, it must be marked as unowned if 2146 * there is zero or one thread only waiting for it. 2147 * Otherwise, it must be marked as contested. 2148 */ 2149 2150 if (count > 1) 2151 new_owner |= UMUTEX_CONTESTED; 2152 again: 2153 error = casueword32(&m->m_owner, owner, &old, new_owner); 2154 if (error == 1) { 2155 error = umtxq_check_susp(td, false); 2156 if (error == 0) 2157 goto again; 2158 } 2159 umtxq_unbusy_unlocked(&key); 2160 umtx_key_release(&key); 2161 if (error == -1) 2162 return (EFAULT); 2163 if (error == 0 && old != owner) 2164 return (EINVAL); 2165 return (error); 2166 } 2167 2168 /* 2169 * Lock a PP mutex. 2170 */ 2171 static int 2172 do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, 2173 struct _umtx_time *timeout, int try) 2174 { 2175 struct abs_timeout timo; 2176 struct umtx_q *uq, *uq2; 2177 struct umtx_pi *pi; 2178 uint32_t ceiling; 2179 uint32_t owner, id; 2180 int error, pri, old_inherited_pri, su, rv; 2181 2182 id = td->td_tid; 2183 uq = td->td_umtxq; 2184 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2185 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), 2186 &uq->uq_key)) != 0) 2187 return (error); 2188 2189 if (timeout != NULL) 2190 abs_timeout_init2(&timo, timeout); 2191 2192 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 2193 for (;;) { 2194 old_inherited_pri = uq->uq_inherited_pri; 2195 umtxq_lock(&uq->uq_key); 2196 umtxq_busy(&uq->uq_key); 2197 umtxq_unlock(&uq->uq_key); 2198 2199 rv = fueword32(&m->m_ceilings[0], &ceiling); 2200 if (rv == -1) { 2201 error = EFAULT; 2202 goto out; 2203 } 2204 ceiling = RTP_PRIO_MAX - ceiling; 2205 if (ceiling > RTP_PRIO_MAX) { 2206 error = EINVAL; 2207 goto out; 2208 } 2209 2210 mtx_lock(&umtx_lock); 2211 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { 2212 mtx_unlock(&umtx_lock); 2213 error = EINVAL; 2214 goto out; 2215 } 2216 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { 2217 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; 2218 thread_lock(td); 2219 if (uq->uq_inherited_pri < UPRI(td)) 2220 sched_lend_user_prio(td, uq->uq_inherited_pri); 2221 thread_unlock(td); 2222 } 2223 mtx_unlock(&umtx_lock); 2224 2225 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, 2226 id | UMUTEX_CONTESTED); 2227 /* The address was invalid. */ 2228 if (rv == -1) { 2229 error = EFAULT; 2230 break; 2231 } 2232 if (rv == 0) { 2233 MPASS(owner == UMUTEX_CONTESTED); 2234 error = 0; 2235 break; 2236 } 2237 /* rv == 1 */ 2238 if (owner == UMUTEX_RB_OWNERDEAD) { 2239 rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD, 2240 &owner, id | UMUTEX_CONTESTED); 2241 if (rv == -1) { 2242 error = EFAULT; 2243 break; 2244 } 2245 if (rv == 0) { 2246 MPASS(owner == UMUTEX_RB_OWNERDEAD); 2247 error = EOWNERDEAD; /* success */ 2248 break; 2249 } 2250 2251 /* 2252 * rv == 1, only check for suspension if we 2253 * did not already catched a signal. If we 2254 * get an error from the check, the same 2255 * condition is checked by the umtxq_sleep() 2256 * call below, so we should obliterate the 2257 * error to not skip the last loop iteration. 2258 */ 2259 if (error == 0) { 2260 error = umtxq_check_susp(td, false); 2261 if (error == 0) { 2262 if (try != 0) 2263 error = EBUSY; 2264 else 2265 continue; 2266 } 2267 error = 0; 2268 } 2269 } else if (owner == UMUTEX_RB_NOTRECOV) { 2270 error = ENOTRECOVERABLE; 2271 } 2272 2273 if (try != 0) 2274 error = EBUSY; 2275 2276 /* 2277 * If we caught a signal, we have retried and now 2278 * exit immediately. 2279 */ 2280 if (error != 0) 2281 break; 2282 2283 umtxq_lock(&uq->uq_key); 2284 umtxq_insert(uq); 2285 umtxq_unbusy(&uq->uq_key); 2286 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ? 2287 NULL : &timo); 2288 umtxq_remove(uq); 2289 umtxq_unlock(&uq->uq_key); 2290 2291 mtx_lock(&umtx_lock); 2292 uq->uq_inherited_pri = old_inherited_pri; 2293 pri = PRI_MAX; 2294 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2295 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2296 if (uq2 != NULL) { 2297 if (pri > UPRI(uq2->uq_thread)) 2298 pri = UPRI(uq2->uq_thread); 2299 } 2300 } 2301 if (pri > uq->uq_inherited_pri) 2302 pri = uq->uq_inherited_pri; 2303 thread_lock(td); 2304 sched_lend_user_prio(td, pri); 2305 thread_unlock(td); 2306 mtx_unlock(&umtx_lock); 2307 } 2308 2309 if (error != 0 && error != EOWNERDEAD) { 2310 mtx_lock(&umtx_lock); 2311 uq->uq_inherited_pri = old_inherited_pri; 2312 pri = PRI_MAX; 2313 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2314 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2315 if (uq2 != NULL) { 2316 if (pri > UPRI(uq2->uq_thread)) 2317 pri = UPRI(uq2->uq_thread); 2318 } 2319 } 2320 if (pri > uq->uq_inherited_pri) 2321 pri = uq->uq_inherited_pri; 2322 thread_lock(td); 2323 sched_lend_user_prio(td, pri); 2324 thread_unlock(td); 2325 mtx_unlock(&umtx_lock); 2326 } 2327 2328 out: 2329 umtxq_unbusy_unlocked(&uq->uq_key); 2330 umtx_key_release(&uq->uq_key); 2331 return (error); 2332 } 2333 2334 /* 2335 * Unlock a PP mutex. 2336 */ 2337 static int 2338 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb) 2339 { 2340 struct umtx_key key; 2341 struct umtx_q *uq, *uq2; 2342 struct umtx_pi *pi; 2343 uint32_t id, owner, rceiling; 2344 int error, pri, new_inherited_pri, su; 2345 2346 id = td->td_tid; 2347 uq = td->td_umtxq; 2348 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 2349 2350 /* 2351 * Make sure we own this mtx. 2352 */ 2353 error = fueword32(&m->m_owner, &owner); 2354 if (error == -1) 2355 return (EFAULT); 2356 2357 if ((owner & ~UMUTEX_CONTESTED) != id) 2358 return (EPERM); 2359 2360 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); 2361 if (error != 0) 2362 return (error); 2363 2364 if (rceiling == -1) 2365 new_inherited_pri = PRI_MAX; 2366 else { 2367 rceiling = RTP_PRIO_MAX - rceiling; 2368 if (rceiling > RTP_PRIO_MAX) 2369 return (EINVAL); 2370 new_inherited_pri = PRI_MIN_REALTIME + rceiling; 2371 } 2372 2373 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2374 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), 2375 &key)) != 0) 2376 return (error); 2377 umtxq_lock(&key); 2378 umtxq_busy(&key); 2379 umtxq_unlock(&key); 2380 /* 2381 * For priority protected mutex, always set unlocked state 2382 * to UMUTEX_CONTESTED, so that userland always enters kernel 2383 * to lock the mutex, it is necessary because thread priority 2384 * has to be adjusted for such mutex. 2385 */ 2386 error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) | 2387 UMUTEX_CONTESTED); 2388 2389 umtxq_lock(&key); 2390 if (error == 0) 2391 umtxq_signal(&key, 1); 2392 umtxq_unbusy(&key); 2393 umtxq_unlock(&key); 2394 2395 if (error == -1) 2396 error = EFAULT; 2397 else { 2398 mtx_lock(&umtx_lock); 2399 if (su != 0) 2400 uq->uq_inherited_pri = new_inherited_pri; 2401 pri = PRI_MAX; 2402 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2403 uq2 = TAILQ_FIRST(&pi->pi_blocked); 2404 if (uq2 != NULL) { 2405 if (pri > UPRI(uq2->uq_thread)) 2406 pri = UPRI(uq2->uq_thread); 2407 } 2408 } 2409 if (pri > uq->uq_inherited_pri) 2410 pri = uq->uq_inherited_pri; 2411 thread_lock(td); 2412 sched_lend_user_prio(td, pri); 2413 thread_unlock(td); 2414 mtx_unlock(&umtx_lock); 2415 } 2416 umtx_key_release(&key); 2417 return (error); 2418 } 2419 2420 static int 2421 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, 2422 uint32_t *old_ceiling) 2423 { 2424 struct umtx_q *uq; 2425 uint32_t flags, id, owner, save_ceiling; 2426 int error, rv, rv1; 2427 2428 error = fueword32(&m->m_flags, &flags); 2429 if (error == -1) 2430 return (EFAULT); 2431 if ((flags & UMUTEX_PRIO_PROTECT) == 0) 2432 return (EINVAL); 2433 if (ceiling > RTP_PRIO_MAX) 2434 return (EINVAL); 2435 id = td->td_tid; 2436 uq = td->td_umtxq; 2437 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? 2438 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), 2439 &uq->uq_key)) != 0) 2440 return (error); 2441 for (;;) { 2442 umtxq_lock(&uq->uq_key); 2443 umtxq_busy(&uq->uq_key); 2444 umtxq_unlock(&uq->uq_key); 2445 2446 rv = fueword32(&m->m_ceilings[0], &save_ceiling); 2447 if (rv == -1) { 2448 error = EFAULT; 2449 break; 2450 } 2451 2452 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, 2453 id | UMUTEX_CONTESTED); 2454 if (rv == -1) { 2455 error = EFAULT; 2456 break; 2457 } 2458 2459 if (rv == 0) { 2460 MPASS(owner == UMUTEX_CONTESTED); 2461 rv = suword32(&m->m_ceilings[0], ceiling); 2462 rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED); 2463 error = (rv == 0 && rv1 == 0) ? 0: EFAULT; 2464 break; 2465 } 2466 2467 if ((owner & ~UMUTEX_CONTESTED) == id) { 2468 rv = suword32(&m->m_ceilings[0], ceiling); 2469 error = rv == 0 ? 0 : EFAULT; 2470 break; 2471 } 2472 2473 if (owner == UMUTEX_RB_OWNERDEAD) { 2474 error = EOWNERDEAD; 2475 break; 2476 } else if (owner == UMUTEX_RB_NOTRECOV) { 2477 error = ENOTRECOVERABLE; 2478 break; 2479 } 2480 2481 /* 2482 * If we caught a signal, we have retried and now 2483 * exit immediately. 2484 */ 2485 if (error != 0) 2486 break; 2487 2488 /* 2489 * We set the contested bit, sleep. Otherwise the lock changed 2490 * and we need to retry or we lost a race to the thread 2491 * unlocking the umtx. 2492 */ 2493 umtxq_lock(&uq->uq_key); 2494 umtxq_insert(uq); 2495 umtxq_unbusy(&uq->uq_key); 2496 error = umtxq_sleep(uq, "umtxpp", NULL); 2497 umtxq_remove(uq); 2498 umtxq_unlock(&uq->uq_key); 2499 } 2500 umtxq_lock(&uq->uq_key); 2501 if (error == 0) 2502 umtxq_signal(&uq->uq_key, INT_MAX); 2503 umtxq_unbusy(&uq->uq_key); 2504 umtxq_unlock(&uq->uq_key); 2505 umtx_key_release(&uq->uq_key); 2506 if (error == 0 && old_ceiling != NULL) { 2507 rv = suword32(old_ceiling, save_ceiling); 2508 error = rv == 0 ? 0 : EFAULT; 2509 } 2510 return (error); 2511 } 2512 2513 /* 2514 * Lock a userland POSIX mutex. 2515 */ 2516 static int 2517 do_lock_umutex(struct thread *td, struct umutex *m, 2518 struct _umtx_time *timeout, int mode) 2519 { 2520 uint32_t flags; 2521 int error; 2522 2523 error = fueword32(&m->m_flags, &flags); 2524 if (error == -1) 2525 return (EFAULT); 2526 2527 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2528 case 0: 2529 error = do_lock_normal(td, m, flags, timeout, mode); 2530 break; 2531 case UMUTEX_PRIO_INHERIT: 2532 error = do_lock_pi(td, m, flags, timeout, mode); 2533 break; 2534 case UMUTEX_PRIO_PROTECT: 2535 error = do_lock_pp(td, m, flags, timeout, mode); 2536 break; 2537 default: 2538 return (EINVAL); 2539 } 2540 if (timeout == NULL) { 2541 if (error == EINTR && mode != _UMUTEX_WAIT) 2542 error = ERESTART; 2543 } else { 2544 /* Timed-locking is not restarted. */ 2545 if (error == ERESTART) 2546 error = EINTR; 2547 } 2548 return (error); 2549 } 2550 2551 /* 2552 * Unlock a userland POSIX mutex. 2553 */ 2554 static int 2555 do_unlock_umutex(struct thread *td, struct umutex *m, bool rb) 2556 { 2557 uint32_t flags; 2558 int error; 2559 2560 error = fueword32(&m->m_flags, &flags); 2561 if (error == -1) 2562 return (EFAULT); 2563 2564 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2565 case 0: 2566 return (do_unlock_normal(td, m, flags, rb)); 2567 case UMUTEX_PRIO_INHERIT: 2568 return (do_unlock_pi(td, m, flags, rb)); 2569 case UMUTEX_PRIO_PROTECT: 2570 return (do_unlock_pp(td, m, flags, rb)); 2571 } 2572 2573 return (EINVAL); 2574 } 2575 2576 static int 2577 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, 2578 struct timespec *timeout, u_long wflags) 2579 { 2580 struct abs_timeout timo; 2581 struct umtx_q *uq; 2582 uint32_t flags, clockid, hasw; 2583 int error; 2584 2585 uq = td->td_umtxq; 2586 error = fueword32(&cv->c_flags, &flags); 2587 if (error == -1) 2588 return (EFAULT); 2589 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); 2590 if (error != 0) 2591 return (error); 2592 2593 if ((wflags & CVWAIT_CLOCKID) != 0) { 2594 error = fueword32(&cv->c_clockid, &clockid); 2595 if (error == -1) { 2596 umtx_key_release(&uq->uq_key); 2597 return (EFAULT); 2598 } 2599 if (clockid < CLOCK_REALTIME || 2600 clockid >= CLOCK_THREAD_CPUTIME_ID) { 2601 /* hmm, only HW clock id will work. */ 2602 umtx_key_release(&uq->uq_key); 2603 return (EINVAL); 2604 } 2605 } else { 2606 clockid = CLOCK_REALTIME; 2607 } 2608 2609 umtxq_lock(&uq->uq_key); 2610 umtxq_busy(&uq->uq_key); 2611 umtxq_insert(uq); 2612 umtxq_unlock(&uq->uq_key); 2613 2614 /* 2615 * Set c_has_waiters to 1 before releasing user mutex, also 2616 * don't modify cache line when unnecessary. 2617 */ 2618 error = fueword32(&cv->c_has_waiters, &hasw); 2619 if (error == 0 && hasw == 0) 2620 suword32(&cv->c_has_waiters, 1); 2621 2622 umtxq_unbusy_unlocked(&uq->uq_key); 2623 2624 error = do_unlock_umutex(td, m, false); 2625 2626 if (timeout != NULL) 2627 abs_timeout_init(&timo, clockid, (wflags & CVWAIT_ABSTIME) != 0, 2628 timeout); 2629 2630 umtxq_lock(&uq->uq_key); 2631 if (error == 0) { 2632 error = umtxq_sleep(uq, "ucond", timeout == NULL ? 2633 NULL : &timo); 2634 } 2635 2636 if ((uq->uq_flags & UQF_UMTXQ) == 0) 2637 error = 0; 2638 else { 2639 /* 2640 * This must be timeout,interrupted by signal or 2641 * surprious wakeup, clear c_has_waiter flag when 2642 * necessary. 2643 */ 2644 umtxq_busy(&uq->uq_key); 2645 if ((uq->uq_flags & UQF_UMTXQ) != 0) { 2646 int oldlen = uq->uq_cur_queue->length; 2647 umtxq_remove(uq); 2648 if (oldlen == 1) { 2649 umtxq_unlock(&uq->uq_key); 2650 suword32(&cv->c_has_waiters, 0); 2651 umtxq_lock(&uq->uq_key); 2652 } 2653 } 2654 umtxq_unbusy(&uq->uq_key); 2655 if (error == ERESTART) 2656 error = EINTR; 2657 } 2658 2659 umtxq_unlock(&uq->uq_key); 2660 umtx_key_release(&uq->uq_key); 2661 return (error); 2662 } 2663 2664 /* 2665 * Signal a userland condition variable. 2666 */ 2667 static int 2668 do_cv_signal(struct thread *td, struct ucond *cv) 2669 { 2670 struct umtx_key key; 2671 int error, cnt, nwake; 2672 uint32_t flags; 2673 2674 error = fueword32(&cv->c_flags, &flags); 2675 if (error == -1) 2676 return (EFAULT); 2677 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2678 return (error); 2679 umtxq_lock(&key); 2680 umtxq_busy(&key); 2681 cnt = umtxq_count(&key); 2682 nwake = umtxq_signal(&key, 1); 2683 if (cnt <= nwake) { 2684 umtxq_unlock(&key); 2685 error = suword32(&cv->c_has_waiters, 0); 2686 if (error == -1) 2687 error = EFAULT; 2688 umtxq_lock(&key); 2689 } 2690 umtxq_unbusy(&key); 2691 umtxq_unlock(&key); 2692 umtx_key_release(&key); 2693 return (error); 2694 } 2695 2696 static int 2697 do_cv_broadcast(struct thread *td, struct ucond *cv) 2698 { 2699 struct umtx_key key; 2700 int error; 2701 uint32_t flags; 2702 2703 error = fueword32(&cv->c_flags, &flags); 2704 if (error == -1) 2705 return (EFAULT); 2706 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2707 return (error); 2708 2709 umtxq_lock(&key); 2710 umtxq_busy(&key); 2711 umtxq_signal(&key, INT_MAX); 2712 umtxq_unlock(&key); 2713 2714 error = suword32(&cv->c_has_waiters, 0); 2715 if (error == -1) 2716 error = EFAULT; 2717 2718 umtxq_unbusy_unlocked(&key); 2719 2720 umtx_key_release(&key); 2721 return (error); 2722 } 2723 2724 static int 2725 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, 2726 struct _umtx_time *timeout) 2727 { 2728 struct abs_timeout timo; 2729 struct umtx_q *uq; 2730 uint32_t flags, wrflags; 2731 int32_t state, oldstate; 2732 int32_t blocked_readers; 2733 int error, error1, rv; 2734 2735 uq = td->td_umtxq; 2736 error = fueword32(&rwlock->rw_flags, &flags); 2737 if (error == -1) 2738 return (EFAULT); 2739 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2740 if (error != 0) 2741 return (error); 2742 2743 if (timeout != NULL) 2744 abs_timeout_init2(&timo, timeout); 2745 2746 wrflags = URWLOCK_WRITE_OWNER; 2747 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER)) 2748 wrflags |= URWLOCK_WRITE_WAITERS; 2749 2750 for (;;) { 2751 rv = fueword32(&rwlock->rw_state, &state); 2752 if (rv == -1) { 2753 umtx_key_release(&uq->uq_key); 2754 return (EFAULT); 2755 } 2756 2757 /* try to lock it */ 2758 while (!(state & wrflags)) { 2759 if (__predict_false(URWLOCK_READER_COUNT(state) == 2760 URWLOCK_MAX_READERS)) { 2761 umtx_key_release(&uq->uq_key); 2762 return (EAGAIN); 2763 } 2764 rv = casueword32(&rwlock->rw_state, state, 2765 &oldstate, state + 1); 2766 if (rv == -1) { 2767 umtx_key_release(&uq->uq_key); 2768 return (EFAULT); 2769 } 2770 if (rv == 0) { 2771 MPASS(oldstate == state); 2772 umtx_key_release(&uq->uq_key); 2773 return (0); 2774 } 2775 error = umtxq_check_susp(td, true); 2776 if (error != 0) 2777 break; 2778 state = oldstate; 2779 } 2780 2781 if (error) 2782 break; 2783 2784 /* grab monitor lock */ 2785 umtxq_lock(&uq->uq_key); 2786 umtxq_busy(&uq->uq_key); 2787 umtxq_unlock(&uq->uq_key); 2788 2789 /* 2790 * re-read the state, in case it changed between the try-lock above 2791 * and the check below 2792 */ 2793 rv = fueword32(&rwlock->rw_state, &state); 2794 if (rv == -1) 2795 error = EFAULT; 2796 2797 /* set read contention bit */ 2798 while (error == 0 && (state & wrflags) && 2799 !(state & URWLOCK_READ_WAITERS)) { 2800 rv = casueword32(&rwlock->rw_state, state, 2801 &oldstate, state | URWLOCK_READ_WAITERS); 2802 if (rv == -1) { 2803 error = EFAULT; 2804 break; 2805 } 2806 if (rv == 0) { 2807 MPASS(oldstate == state); 2808 goto sleep; 2809 } 2810 state = oldstate; 2811 error = umtxq_check_susp(td, false); 2812 if (error != 0) 2813 break; 2814 } 2815 if (error != 0) { 2816 umtxq_unbusy_unlocked(&uq->uq_key); 2817 break; 2818 } 2819 2820 /* state is changed while setting flags, restart */ 2821 if (!(state & wrflags)) { 2822 umtxq_unbusy_unlocked(&uq->uq_key); 2823 error = umtxq_check_susp(td, true); 2824 if (error != 0) 2825 break; 2826 continue; 2827 } 2828 2829 sleep: 2830 /* 2831 * Contention bit is set, before sleeping, increase 2832 * read waiter count. 2833 */ 2834 rv = fueword32(&rwlock->rw_blocked_readers, 2835 &blocked_readers); 2836 if (rv == -1) { 2837 umtxq_unbusy_unlocked(&uq->uq_key); 2838 error = EFAULT; 2839 break; 2840 } 2841 suword32(&rwlock->rw_blocked_readers, blocked_readers+1); 2842 2843 while (state & wrflags) { 2844 umtxq_lock(&uq->uq_key); 2845 umtxq_insert(uq); 2846 umtxq_unbusy(&uq->uq_key); 2847 2848 error = umtxq_sleep(uq, "urdlck", timeout == NULL ? 2849 NULL : &timo); 2850 2851 umtxq_busy(&uq->uq_key); 2852 umtxq_remove(uq); 2853 umtxq_unlock(&uq->uq_key); 2854 if (error) 2855 break; 2856 rv = fueword32(&rwlock->rw_state, &state); 2857 if (rv == -1) { 2858 error = EFAULT; 2859 break; 2860 } 2861 } 2862 2863 /* decrease read waiter count, and may clear read contention bit */ 2864 rv = fueword32(&rwlock->rw_blocked_readers, 2865 &blocked_readers); 2866 if (rv == -1) { 2867 umtxq_unbusy_unlocked(&uq->uq_key); 2868 error = EFAULT; 2869 break; 2870 } 2871 suword32(&rwlock->rw_blocked_readers, blocked_readers-1); 2872 if (blocked_readers == 1) { 2873 rv = fueword32(&rwlock->rw_state, &state); 2874 if (rv == -1) { 2875 umtxq_unbusy_unlocked(&uq->uq_key); 2876 error = EFAULT; 2877 break; 2878 } 2879 for (;;) { 2880 rv = casueword32(&rwlock->rw_state, state, 2881 &oldstate, state & ~URWLOCK_READ_WAITERS); 2882 if (rv == -1) { 2883 error = EFAULT; 2884 break; 2885 } 2886 if (rv == 0) { 2887 MPASS(oldstate == state); 2888 break; 2889 } 2890 state = oldstate; 2891 error1 = umtxq_check_susp(td, false); 2892 if (error1 != 0) { 2893 if (error == 0) 2894 error = error1; 2895 break; 2896 } 2897 } 2898 } 2899 2900 umtxq_unbusy_unlocked(&uq->uq_key); 2901 if (error != 0) 2902 break; 2903 } 2904 umtx_key_release(&uq->uq_key); 2905 if (error == ERESTART) 2906 error = EINTR; 2907 return (error); 2908 } 2909 2910 static int 2911 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout) 2912 { 2913 struct abs_timeout timo; 2914 struct umtx_q *uq; 2915 uint32_t flags; 2916 int32_t state, oldstate; 2917 int32_t blocked_writers; 2918 int32_t blocked_readers; 2919 int error, error1, rv; 2920 2921 uq = td->td_umtxq; 2922 error = fueword32(&rwlock->rw_flags, &flags); 2923 if (error == -1) 2924 return (EFAULT); 2925 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2926 if (error != 0) 2927 return (error); 2928 2929 if (timeout != NULL) 2930 abs_timeout_init2(&timo, timeout); 2931 2932 blocked_readers = 0; 2933 for (;;) { 2934 rv = fueword32(&rwlock->rw_state, &state); 2935 if (rv == -1) { 2936 umtx_key_release(&uq->uq_key); 2937 return (EFAULT); 2938 } 2939 while ((state & URWLOCK_WRITE_OWNER) == 0 && 2940 URWLOCK_READER_COUNT(state) == 0) { 2941 rv = casueword32(&rwlock->rw_state, state, 2942 &oldstate, state | URWLOCK_WRITE_OWNER); 2943 if (rv == -1) { 2944 umtx_key_release(&uq->uq_key); 2945 return (EFAULT); 2946 } 2947 if (rv == 0) { 2948 MPASS(oldstate == state); 2949 umtx_key_release(&uq->uq_key); 2950 return (0); 2951 } 2952 state = oldstate; 2953 error = umtxq_check_susp(td, true); 2954 if (error != 0) 2955 break; 2956 } 2957 2958 if (error) { 2959 if ((state & (URWLOCK_WRITE_OWNER | 2960 URWLOCK_WRITE_WAITERS)) == 0 && 2961 blocked_readers != 0) { 2962 umtxq_lock(&uq->uq_key); 2963 umtxq_busy(&uq->uq_key); 2964 umtxq_signal_queue(&uq->uq_key, INT_MAX, 2965 UMTX_SHARED_QUEUE); 2966 umtxq_unbusy(&uq->uq_key); 2967 umtxq_unlock(&uq->uq_key); 2968 } 2969 2970 break; 2971 } 2972 2973 /* grab monitor lock */ 2974 umtxq_lock(&uq->uq_key); 2975 umtxq_busy(&uq->uq_key); 2976 umtxq_unlock(&uq->uq_key); 2977 2978 /* 2979 * Re-read the state, in case it changed between the 2980 * try-lock above and the check below. 2981 */ 2982 rv = fueword32(&rwlock->rw_state, &state); 2983 if (rv == -1) 2984 error = EFAULT; 2985 2986 while (error == 0 && ((state & URWLOCK_WRITE_OWNER) || 2987 URWLOCK_READER_COUNT(state) != 0) && 2988 (state & URWLOCK_WRITE_WAITERS) == 0) { 2989 rv = casueword32(&rwlock->rw_state, state, 2990 &oldstate, state | URWLOCK_WRITE_WAITERS); 2991 if (rv == -1) { 2992 error = EFAULT; 2993 break; 2994 } 2995 if (rv == 0) { 2996 MPASS(oldstate == state); 2997 goto sleep; 2998 } 2999 state = oldstate; 3000 error = umtxq_check_susp(td, false); 3001 if (error != 0) 3002 break; 3003 } 3004 if (error != 0) { 3005 umtxq_unbusy_unlocked(&uq->uq_key); 3006 break; 3007 } 3008 3009 if ((state & URWLOCK_WRITE_OWNER) == 0 && 3010 URWLOCK_READER_COUNT(state) == 0) { 3011 umtxq_unbusy_unlocked(&uq->uq_key); 3012 error = umtxq_check_susp(td, false); 3013 if (error != 0) 3014 break; 3015 continue; 3016 } 3017 sleep: 3018 rv = fueword32(&rwlock->rw_blocked_writers, 3019 &blocked_writers); 3020 if (rv == -1) { 3021 umtxq_unbusy_unlocked(&uq->uq_key); 3022 error = EFAULT; 3023 break; 3024 } 3025 suword32(&rwlock->rw_blocked_writers, blocked_writers + 1); 3026 3027 while ((state & URWLOCK_WRITE_OWNER) || 3028 URWLOCK_READER_COUNT(state) != 0) { 3029 umtxq_lock(&uq->uq_key); 3030 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE); 3031 umtxq_unbusy(&uq->uq_key); 3032 3033 error = umtxq_sleep(uq, "uwrlck", timeout == NULL ? 3034 NULL : &timo); 3035 3036 umtxq_busy(&uq->uq_key); 3037 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE); 3038 umtxq_unlock(&uq->uq_key); 3039 if (error) 3040 break; 3041 rv = fueword32(&rwlock->rw_state, &state); 3042 if (rv == -1) { 3043 error = EFAULT; 3044 break; 3045 } 3046 } 3047 3048 rv = fueword32(&rwlock->rw_blocked_writers, 3049 &blocked_writers); 3050 if (rv == -1) { 3051 umtxq_unbusy_unlocked(&uq->uq_key); 3052 error = EFAULT; 3053 break; 3054 } 3055 suword32(&rwlock->rw_blocked_writers, blocked_writers-1); 3056 if (blocked_writers == 1) { 3057 rv = fueword32(&rwlock->rw_state, &state); 3058 if (rv == -1) { 3059 umtxq_unbusy_unlocked(&uq->uq_key); 3060 error = EFAULT; 3061 break; 3062 } 3063 for (;;) { 3064 rv = casueword32(&rwlock->rw_state, state, 3065 &oldstate, state & ~URWLOCK_WRITE_WAITERS); 3066 if (rv == -1) { 3067 error = EFAULT; 3068 break; 3069 } 3070 if (rv == 0) { 3071 MPASS(oldstate == state); 3072 break; 3073 } 3074 state = oldstate; 3075 error1 = umtxq_check_susp(td, false); 3076 /* 3077 * We are leaving the URWLOCK_WRITE_WAITERS 3078 * behind, but this should not harm the 3079 * correctness. 3080 */ 3081 if (error1 != 0) { 3082 if (error == 0) 3083 error = error1; 3084 break; 3085 } 3086 } 3087 rv = fueword32(&rwlock->rw_blocked_readers, 3088 &blocked_readers); 3089 if (rv == -1) { 3090 umtxq_unbusy_unlocked(&uq->uq_key); 3091 error = EFAULT; 3092 break; 3093 } 3094 } else 3095 blocked_readers = 0; 3096 3097 umtxq_unbusy_unlocked(&uq->uq_key); 3098 } 3099 3100 umtx_key_release(&uq->uq_key); 3101 if (error == ERESTART) 3102 error = EINTR; 3103 return (error); 3104 } 3105 3106 static int 3107 do_rw_unlock(struct thread *td, struct urwlock *rwlock) 3108 { 3109 struct umtx_q *uq; 3110 uint32_t flags; 3111 int32_t state, oldstate; 3112 int error, rv, q, count; 3113 3114 uq = td->td_umtxq; 3115 error = fueword32(&rwlock->rw_flags, &flags); 3116 if (error == -1) 3117 return (EFAULT); 3118 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 3119 if (error != 0) 3120 return (error); 3121 3122 error = fueword32(&rwlock->rw_state, &state); 3123 if (error == -1) { 3124 error = EFAULT; 3125 goto out; 3126 } 3127 if (state & URWLOCK_WRITE_OWNER) { 3128 for (;;) { 3129 rv = casueword32(&rwlock->rw_state, state, 3130 &oldstate, state & ~URWLOCK_WRITE_OWNER); 3131 if (rv == -1) { 3132 error = EFAULT; 3133 goto out; 3134 } 3135 if (rv == 1) { 3136 state = oldstate; 3137 if (!(oldstate & URWLOCK_WRITE_OWNER)) { 3138 error = EPERM; 3139 goto out; 3140 } 3141 error = umtxq_check_susp(td, true); 3142 if (error != 0) 3143 goto out; 3144 } else 3145 break; 3146 } 3147 } else if (URWLOCK_READER_COUNT(state) != 0) { 3148 for (;;) { 3149 rv = casueword32(&rwlock->rw_state, state, 3150 &oldstate, state - 1); 3151 if (rv == -1) { 3152 error = EFAULT; 3153 goto out; 3154 } 3155 if (rv == 1) { 3156 state = oldstate; 3157 if (URWLOCK_READER_COUNT(oldstate) == 0) { 3158 error = EPERM; 3159 goto out; 3160 } 3161 error = umtxq_check_susp(td, true); 3162 if (error != 0) 3163 goto out; 3164 } else 3165 break; 3166 } 3167 } else { 3168 error = EPERM; 3169 goto out; 3170 } 3171 3172 count = 0; 3173 3174 if (!(flags & URWLOCK_PREFER_READER)) { 3175 if (state & URWLOCK_WRITE_WAITERS) { 3176 count = 1; 3177 q = UMTX_EXCLUSIVE_QUEUE; 3178 } else if (state & URWLOCK_READ_WAITERS) { 3179 count = INT_MAX; 3180 q = UMTX_SHARED_QUEUE; 3181 } 3182 } else { 3183 if (state & URWLOCK_READ_WAITERS) { 3184 count = INT_MAX; 3185 q = UMTX_SHARED_QUEUE; 3186 } else if (state & URWLOCK_WRITE_WAITERS) { 3187 count = 1; 3188 q = UMTX_EXCLUSIVE_QUEUE; 3189 } 3190 } 3191 3192 if (count) { 3193 umtxq_lock(&uq->uq_key); 3194 umtxq_busy(&uq->uq_key); 3195 umtxq_signal_queue(&uq->uq_key, count, q); 3196 umtxq_unbusy(&uq->uq_key); 3197 umtxq_unlock(&uq->uq_key); 3198 } 3199 out: 3200 umtx_key_release(&uq->uq_key); 3201 return (error); 3202 } 3203 3204 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 3205 static int 3206 do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) 3207 { 3208 struct abs_timeout timo; 3209 struct umtx_q *uq; 3210 uint32_t flags, count, count1; 3211 int error, rv, rv1; 3212 3213 uq = td->td_umtxq; 3214 error = fueword32(&sem->_flags, &flags); 3215 if (error == -1) 3216 return (EFAULT); 3217 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); 3218 if (error != 0) 3219 return (error); 3220 3221 if (timeout != NULL) 3222 abs_timeout_init2(&timo, timeout); 3223 3224 again: 3225 umtxq_lock(&uq->uq_key); 3226 umtxq_busy(&uq->uq_key); 3227 umtxq_insert(uq); 3228 umtxq_unlock(&uq->uq_key); 3229 rv = casueword32(&sem->_has_waiters, 0, &count1, 1); 3230 if (rv == 0) 3231 rv1 = fueword32(&sem->_count, &count); 3232 if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || rv == 1) { 3233 umtxq_lock(&uq->uq_key); 3234 umtxq_unbusy(&uq->uq_key); 3235 umtxq_remove(uq); 3236 umtxq_unlock(&uq->uq_key); 3237 if (rv == 1) { 3238 rv = umtxq_check_susp(td, true); 3239 if (rv == 0) 3240 goto again; 3241 error = rv; 3242 goto out; 3243 } 3244 if (rv == 0) 3245 rv = rv1; 3246 error = rv == -1 ? EFAULT : 0; 3247 goto out; 3248 } 3249 umtxq_lock(&uq->uq_key); 3250 umtxq_unbusy(&uq->uq_key); 3251 3252 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); 3253 3254 if ((uq->uq_flags & UQF_UMTXQ) == 0) 3255 error = 0; 3256 else { 3257 umtxq_remove(uq); 3258 /* A relative timeout cannot be restarted. */ 3259 if (error == ERESTART && timeout != NULL && 3260 (timeout->_flags & UMTX_ABSTIME) == 0) 3261 error = EINTR; 3262 } 3263 umtxq_unlock(&uq->uq_key); 3264 out: 3265 umtx_key_release(&uq->uq_key); 3266 return (error); 3267 } 3268 3269 /* 3270 * Signal a userland semaphore. 3271 */ 3272 static int 3273 do_sem_wake(struct thread *td, struct _usem *sem) 3274 { 3275 struct umtx_key key; 3276 int error, cnt; 3277 uint32_t flags; 3278 3279 error = fueword32(&sem->_flags, &flags); 3280 if (error == -1) 3281 return (EFAULT); 3282 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) 3283 return (error); 3284 umtxq_lock(&key); 3285 umtxq_busy(&key); 3286 cnt = umtxq_count(&key); 3287 if (cnt > 0) { 3288 /* 3289 * Check if count is greater than 0, this means the memory is 3290 * still being referenced by user code, so we can safely 3291 * update _has_waiters flag. 3292 */ 3293 if (cnt == 1) { 3294 umtxq_unlock(&key); 3295 error = suword32(&sem->_has_waiters, 0); 3296 umtxq_lock(&key); 3297 if (error == -1) 3298 error = EFAULT; 3299 } 3300 umtxq_signal(&key, 1); 3301 } 3302 umtxq_unbusy(&key); 3303 umtxq_unlock(&key); 3304 umtx_key_release(&key); 3305 return (error); 3306 } 3307 #endif 3308 3309 static int 3310 do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) 3311 { 3312 struct abs_timeout timo; 3313 struct umtx_q *uq; 3314 uint32_t count, flags; 3315 int error, rv; 3316 3317 uq = td->td_umtxq; 3318 flags = fuword32(&sem->_flags); 3319 if (timeout != NULL) 3320 abs_timeout_init2(&timo, timeout); 3321 3322 again: 3323 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); 3324 if (error != 0) 3325 return (error); 3326 umtxq_lock(&uq->uq_key); 3327 umtxq_busy(&uq->uq_key); 3328 umtxq_insert(uq); 3329 umtxq_unlock(&uq->uq_key); 3330 rv = fueword32(&sem->_count, &count); 3331 if (rv == -1) { 3332 umtxq_lock(&uq->uq_key); 3333 umtxq_unbusy(&uq->uq_key); 3334 umtxq_remove(uq); 3335 umtxq_unlock(&uq->uq_key); 3336 umtx_key_release(&uq->uq_key); 3337 return (EFAULT); 3338 } 3339 for (;;) { 3340 if (USEM_COUNT(count) != 0) { 3341 umtxq_lock(&uq->uq_key); 3342 umtxq_unbusy(&uq->uq_key); 3343 umtxq_remove(uq); 3344 umtxq_unlock(&uq->uq_key); 3345 umtx_key_release(&uq->uq_key); 3346 return (0); 3347 } 3348 if (count == USEM_HAS_WAITERS) 3349 break; 3350 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); 3351 if (rv == 0) 3352 break; 3353 umtxq_lock(&uq->uq_key); 3354 umtxq_unbusy(&uq->uq_key); 3355 umtxq_remove(uq); 3356 umtxq_unlock(&uq->uq_key); 3357 umtx_key_release(&uq->uq_key); 3358 if (rv == -1) 3359 return (EFAULT); 3360 rv = umtxq_check_susp(td, true); 3361 if (rv != 0) 3362 return (rv); 3363 goto again; 3364 } 3365 umtxq_lock(&uq->uq_key); 3366 umtxq_unbusy(&uq->uq_key); 3367 3368 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); 3369 3370 if ((uq->uq_flags & UQF_UMTXQ) == 0) 3371 error = 0; 3372 else { 3373 umtxq_remove(uq); 3374 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) { 3375 /* A relative timeout cannot be restarted. */ 3376 if (error == ERESTART) 3377 error = EINTR; 3378 if (error == EINTR) { 3379 abs_timeout_update(&timo); 3380 timespecsub(&timo.end, &timo.cur, 3381 &timeout->_timeout); 3382 } 3383 } 3384 } 3385 umtxq_unlock(&uq->uq_key); 3386 umtx_key_release(&uq->uq_key); 3387 return (error); 3388 } 3389 3390 /* 3391 * Signal a userland semaphore. 3392 */ 3393 static int 3394 do_sem2_wake(struct thread *td, struct _usem2 *sem) 3395 { 3396 struct umtx_key key; 3397 int error, cnt, rv; 3398 uint32_t count, flags; 3399 3400 rv = fueword32(&sem->_flags, &flags); 3401 if (rv == -1) 3402 return (EFAULT); 3403 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) 3404 return (error); 3405 umtxq_lock(&key); 3406 umtxq_busy(&key); 3407 cnt = umtxq_count(&key); 3408 if (cnt > 0) { 3409 /* 3410 * If this was the last sleeping thread, clear the waiters 3411 * flag in _count. 3412 */ 3413 if (cnt == 1) { 3414 umtxq_unlock(&key); 3415 rv = fueword32(&sem->_count, &count); 3416 while (rv != -1 && count & USEM_HAS_WAITERS) { 3417 rv = casueword32(&sem->_count, count, &count, 3418 count & ~USEM_HAS_WAITERS); 3419 if (rv == 1) { 3420 rv = umtxq_check_susp(td, true); 3421 if (rv != 0) 3422 break; 3423 } 3424 } 3425 if (rv == -1) 3426 error = EFAULT; 3427 else if (rv > 0) { 3428 error = rv; 3429 } 3430 umtxq_lock(&key); 3431 } 3432 3433 umtxq_signal(&key, 1); 3434 } 3435 umtxq_unbusy(&key); 3436 umtxq_unlock(&key); 3437 umtx_key_release(&key); 3438 return (error); 3439 } 3440 3441 inline int 3442 umtx_copyin_timeout(const void *addr, struct timespec *tsp) 3443 { 3444 int error; 3445 3446 error = copyin(addr, tsp, sizeof(struct timespec)); 3447 if (error == 0) { 3448 if (tsp->tv_sec < 0 || 3449 tsp->tv_nsec >= 1000000000 || 3450 tsp->tv_nsec < 0) 3451 error = EINVAL; 3452 } 3453 return (error); 3454 } 3455 3456 static inline int 3457 umtx_copyin_umtx_time(const void *addr, size_t size, struct _umtx_time *tp) 3458 { 3459 int error; 3460 3461 if (size <= sizeof(struct timespec)) { 3462 tp->_clockid = CLOCK_REALTIME; 3463 tp->_flags = 0; 3464 error = copyin(addr, &tp->_timeout, sizeof(struct timespec)); 3465 } else 3466 error = copyin(addr, tp, sizeof(struct _umtx_time)); 3467 if (error != 0) 3468 return (error); 3469 if (tp->_timeout.tv_sec < 0 || 3470 tp->_timeout.tv_nsec >= 1000000000 || tp->_timeout.tv_nsec < 0) 3471 return (EINVAL); 3472 return (0); 3473 } 3474 3475 static int 3476 __umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap) 3477 { 3478 3479 return (EOPNOTSUPP); 3480 } 3481 3482 static int 3483 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) 3484 { 3485 struct _umtx_time timeout, *tm_p; 3486 int error; 3487 3488 if (uap->uaddr2 == NULL) 3489 tm_p = NULL; 3490 else { 3491 error = umtx_copyin_umtx_time( 3492 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3493 if (error != 0) 3494 return (error); 3495 tm_p = &timeout; 3496 } 3497 return (do_wait(td, uap->obj, uap->val, tm_p, 0, 0)); 3498 } 3499 3500 static int 3501 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap) 3502 { 3503 struct _umtx_time timeout, *tm_p; 3504 int error; 3505 3506 if (uap->uaddr2 == NULL) 3507 tm_p = NULL; 3508 else { 3509 error = umtx_copyin_umtx_time( 3510 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3511 if (error != 0) 3512 return (error); 3513 tm_p = &timeout; 3514 } 3515 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); 3516 } 3517 3518 static int 3519 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) 3520 { 3521 struct _umtx_time *tm_p, timeout; 3522 int error; 3523 3524 if (uap->uaddr2 == NULL) 3525 tm_p = NULL; 3526 else { 3527 error = umtx_copyin_umtx_time( 3528 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3529 if (error != 0) 3530 return (error); 3531 tm_p = &timeout; 3532 } 3533 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); 3534 } 3535 3536 static int 3537 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) 3538 { 3539 3540 return (kern_umtx_wake(td, uap->obj, uap->val, 0)); 3541 } 3542 3543 #define BATCH_SIZE 128 3544 static int 3545 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap) 3546 { 3547 char *uaddrs[BATCH_SIZE], **upp; 3548 int count, error, i, pos, tocopy; 3549 3550 upp = (char **)uap->obj; 3551 error = 0; 3552 for (count = uap->val, pos = 0; count > 0; count -= tocopy, 3553 pos += tocopy) { 3554 tocopy = MIN(count, BATCH_SIZE); 3555 error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *)); 3556 if (error != 0) 3557 break; 3558 for (i = 0; i < tocopy; ++i) 3559 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1); 3560 maybe_yield(); 3561 } 3562 return (error); 3563 } 3564 3565 static int 3566 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) 3567 { 3568 3569 return (kern_umtx_wake(td, uap->obj, uap->val, 1)); 3570 } 3571 3572 static int 3573 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) 3574 { 3575 struct _umtx_time *tm_p, timeout; 3576 int error; 3577 3578 /* Allow a null timespec (wait forever). */ 3579 if (uap->uaddr2 == NULL) 3580 tm_p = NULL; 3581 else { 3582 error = umtx_copyin_umtx_time( 3583 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3584 if (error != 0) 3585 return (error); 3586 tm_p = &timeout; 3587 } 3588 return (do_lock_umutex(td, uap->obj, tm_p, 0)); 3589 } 3590 3591 static int 3592 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) 3593 { 3594 3595 return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY)); 3596 } 3597 3598 static int 3599 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) 3600 { 3601 struct _umtx_time *tm_p, timeout; 3602 int error; 3603 3604 /* Allow a null timespec (wait forever). */ 3605 if (uap->uaddr2 == NULL) 3606 tm_p = NULL; 3607 else { 3608 error = umtx_copyin_umtx_time( 3609 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3610 if (error != 0) 3611 return (error); 3612 tm_p = &timeout; 3613 } 3614 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); 3615 } 3616 3617 static int 3618 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) 3619 { 3620 3621 return (do_wake_umutex(td, uap->obj)); 3622 } 3623 3624 static int 3625 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) 3626 { 3627 3628 return (do_unlock_umutex(td, uap->obj, false)); 3629 } 3630 3631 static int 3632 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) 3633 { 3634 3635 return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1)); 3636 } 3637 3638 static int 3639 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) 3640 { 3641 struct timespec *ts, timeout; 3642 int error; 3643 3644 /* Allow a null timespec (wait forever). */ 3645 if (uap->uaddr2 == NULL) 3646 ts = NULL; 3647 else { 3648 error = umtx_copyin_timeout(uap->uaddr2, &timeout); 3649 if (error != 0) 3650 return (error); 3651 ts = &timeout; 3652 } 3653 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 3654 } 3655 3656 static int 3657 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) 3658 { 3659 3660 return (do_cv_signal(td, uap->obj)); 3661 } 3662 3663 static int 3664 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) 3665 { 3666 3667 return (do_cv_broadcast(td, uap->obj)); 3668 } 3669 3670 static int 3671 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap) 3672 { 3673 struct _umtx_time timeout; 3674 int error; 3675 3676 /* Allow a null timespec (wait forever). */ 3677 if (uap->uaddr2 == NULL) { 3678 error = do_rw_rdlock(td, uap->obj, uap->val, 0); 3679 } else { 3680 error = umtx_copyin_umtx_time(uap->uaddr2, 3681 (size_t)uap->uaddr1, &timeout); 3682 if (error != 0) 3683 return (error); 3684 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); 3685 } 3686 return (error); 3687 } 3688 3689 static int 3690 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap) 3691 { 3692 struct _umtx_time timeout; 3693 int error; 3694 3695 /* Allow a null timespec (wait forever). */ 3696 if (uap->uaddr2 == NULL) { 3697 error = do_rw_wrlock(td, uap->obj, 0); 3698 } else { 3699 error = umtx_copyin_umtx_time(uap->uaddr2, 3700 (size_t)uap->uaddr1, &timeout); 3701 if (error != 0) 3702 return (error); 3703 3704 error = do_rw_wrlock(td, uap->obj, &timeout); 3705 } 3706 return (error); 3707 } 3708 3709 static int 3710 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) 3711 { 3712 3713 return (do_rw_unlock(td, uap->obj)); 3714 } 3715 3716 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 3717 static int 3718 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap) 3719 { 3720 struct _umtx_time *tm_p, timeout; 3721 int error; 3722 3723 /* Allow a null timespec (wait forever). */ 3724 if (uap->uaddr2 == NULL) 3725 tm_p = NULL; 3726 else { 3727 error = umtx_copyin_umtx_time( 3728 uap->uaddr2, (size_t)uap->uaddr1, &timeout); 3729 if (error != 0) 3730 return (error); 3731 tm_p = &timeout; 3732 } 3733 return (do_sem_wait(td, uap->obj, tm_p)); 3734 } 3735 3736 static int 3737 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap) 3738 { 3739 3740 return (do_sem_wake(td, uap->obj)); 3741 } 3742 #endif 3743 3744 static int 3745 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap) 3746 { 3747 3748 return (do_wake2_umutex(td, uap->obj, uap->val)); 3749 } 3750 3751 static int 3752 __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap) 3753 { 3754 struct _umtx_time *tm_p, timeout; 3755 size_t uasize; 3756 int error; 3757 3758 /* Allow a null timespec (wait forever). */ 3759 if (uap->uaddr2 == NULL) { 3760 uasize = 0; 3761 tm_p = NULL; 3762 } else { 3763 uasize = (size_t)uap->uaddr1; 3764 error = umtx_copyin_umtx_time(uap->uaddr2, uasize, &timeout); 3765 if (error != 0) 3766 return (error); 3767 tm_p = &timeout; 3768 } 3769 error = do_sem2_wait(td, uap->obj, tm_p); 3770 if (error == EINTR && uap->uaddr2 != NULL && 3771 (timeout._flags & UMTX_ABSTIME) == 0 && 3772 uasize >= sizeof(struct _umtx_time) + sizeof(struct timespec)) { 3773 error = copyout(&timeout._timeout, 3774 (struct _umtx_time *)uap->uaddr2 + 1, 3775 sizeof(struct timespec)); 3776 if (error == 0) { 3777 error = EINTR; 3778 } 3779 } 3780 3781 return (error); 3782 } 3783 3784 static int 3785 __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap) 3786 { 3787 3788 return (do_sem2_wake(td, uap->obj)); 3789 } 3790 3791 #define USHM_OBJ_UMTX(o) \ 3792 ((struct umtx_shm_obj_list *)(&(o)->umtx_data)) 3793 3794 #define USHMF_REG_LINKED 0x0001 3795 #define USHMF_OBJ_LINKED 0x0002 3796 struct umtx_shm_reg { 3797 TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link; 3798 LIST_ENTRY(umtx_shm_reg) ushm_obj_link; 3799 struct umtx_key ushm_key; 3800 struct ucred *ushm_cred; 3801 struct shmfd *ushm_obj; 3802 u_int ushm_refcnt; 3803 u_int ushm_flags; 3804 }; 3805 3806 LIST_HEAD(umtx_shm_obj_list, umtx_shm_reg); 3807 TAILQ_HEAD(umtx_shm_reg_head, umtx_shm_reg); 3808 3809 static uma_zone_t umtx_shm_reg_zone; 3810 static struct umtx_shm_reg_head umtx_shm_registry[UMTX_CHAINS]; 3811 static struct mtx umtx_shm_lock; 3812 static struct umtx_shm_reg_head umtx_shm_reg_delfree = 3813 TAILQ_HEAD_INITIALIZER(umtx_shm_reg_delfree); 3814 3815 static void umtx_shm_free_reg(struct umtx_shm_reg *reg); 3816 3817 static void 3818 umtx_shm_reg_delfree_tq(void *context __unused, int pending __unused) 3819 { 3820 struct umtx_shm_reg_head d; 3821 struct umtx_shm_reg *reg, *reg1; 3822 3823 TAILQ_INIT(&d); 3824 mtx_lock(&umtx_shm_lock); 3825 TAILQ_CONCAT(&d, &umtx_shm_reg_delfree, ushm_reg_link); 3826 mtx_unlock(&umtx_shm_lock); 3827 TAILQ_FOREACH_SAFE(reg, &d, ushm_reg_link, reg1) { 3828 TAILQ_REMOVE(&d, reg, ushm_reg_link); 3829 umtx_shm_free_reg(reg); 3830 } 3831 } 3832 3833 static struct task umtx_shm_reg_delfree_task = 3834 TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL); 3835 3836 static struct umtx_shm_reg * 3837 umtx_shm_find_reg_locked(const struct umtx_key *key) 3838 { 3839 struct umtx_shm_reg *reg; 3840 struct umtx_shm_reg_head *reg_head; 3841 3842 KASSERT(key->shared, ("umtx_p_find_rg: private key")); 3843 mtx_assert(&umtx_shm_lock, MA_OWNED); 3844 reg_head = &umtx_shm_registry[key->hash]; 3845 TAILQ_FOREACH(reg, reg_head, ushm_reg_link) { 3846 KASSERT(reg->ushm_key.shared, 3847 ("non-shared key on reg %p %d", reg, reg->ushm_key.shared)); 3848 if (reg->ushm_key.info.shared.object == 3849 key->info.shared.object && 3850 reg->ushm_key.info.shared.offset == 3851 key->info.shared.offset) { 3852 KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM")); 3853 KASSERT(reg->ushm_refcnt > 0, 3854 ("reg %p refcnt 0 onlist", reg)); 3855 KASSERT((reg->ushm_flags & USHMF_REG_LINKED) != 0, 3856 ("reg %p not linked", reg)); 3857 reg->ushm_refcnt++; 3858 return (reg); 3859 } 3860 } 3861 return (NULL); 3862 } 3863 3864 static struct umtx_shm_reg * 3865 umtx_shm_find_reg(const struct umtx_key *key) 3866 { 3867 struct umtx_shm_reg *reg; 3868 3869 mtx_lock(&umtx_shm_lock); 3870 reg = umtx_shm_find_reg_locked(key); 3871 mtx_unlock(&umtx_shm_lock); 3872 return (reg); 3873 } 3874 3875 static void 3876 umtx_shm_free_reg(struct umtx_shm_reg *reg) 3877 { 3878 3879 chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0); 3880 crfree(reg->ushm_cred); 3881 shm_drop(reg->ushm_obj); 3882 uma_zfree(umtx_shm_reg_zone, reg); 3883 } 3884 3885 static bool 3886 umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool force) 3887 { 3888 bool res; 3889 3890 mtx_assert(&umtx_shm_lock, MA_OWNED); 3891 KASSERT(reg->ushm_refcnt > 0, ("ushm_reg %p refcnt 0", reg)); 3892 reg->ushm_refcnt--; 3893 res = reg->ushm_refcnt == 0; 3894 if (res || force) { 3895 if ((reg->ushm_flags & USHMF_REG_LINKED) != 0) { 3896 TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], 3897 reg, ushm_reg_link); 3898 reg->ushm_flags &= ~USHMF_REG_LINKED; 3899 } 3900 if ((reg->ushm_flags & USHMF_OBJ_LINKED) != 0) { 3901 LIST_REMOVE(reg, ushm_obj_link); 3902 reg->ushm_flags &= ~USHMF_OBJ_LINKED; 3903 } 3904 } 3905 return (res); 3906 } 3907 3908 static void 3909 umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool force) 3910 { 3911 vm_object_t object; 3912 bool dofree; 3913 3914 if (force) { 3915 object = reg->ushm_obj->shm_object; 3916 VM_OBJECT_WLOCK(object); 3917 object->flags |= OBJ_UMTXDEAD; 3918 VM_OBJECT_WUNLOCK(object); 3919 } 3920 mtx_lock(&umtx_shm_lock); 3921 dofree = umtx_shm_unref_reg_locked(reg, force); 3922 mtx_unlock(&umtx_shm_lock); 3923 if (dofree) 3924 umtx_shm_free_reg(reg); 3925 } 3926 3927 void 3928 umtx_shm_object_init(vm_object_t object) 3929 { 3930 3931 LIST_INIT(USHM_OBJ_UMTX(object)); 3932 } 3933 3934 void 3935 umtx_shm_object_terminated(vm_object_t object) 3936 { 3937 struct umtx_shm_reg *reg, *reg1; 3938 bool dofree; 3939 3940 if (LIST_EMPTY(USHM_OBJ_UMTX(object))) 3941 return; 3942 3943 dofree = false; 3944 mtx_lock(&umtx_shm_lock); 3945 LIST_FOREACH_SAFE(reg, USHM_OBJ_UMTX(object), ushm_obj_link, reg1) { 3946 if (umtx_shm_unref_reg_locked(reg, true)) { 3947 TAILQ_INSERT_TAIL(&umtx_shm_reg_delfree, reg, 3948 ushm_reg_link); 3949 dofree = true; 3950 } 3951 } 3952 mtx_unlock(&umtx_shm_lock); 3953 if (dofree) 3954 taskqueue_enqueue(taskqueue_thread, &umtx_shm_reg_delfree_task); 3955 } 3956 3957 static int 3958 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key, 3959 struct umtx_shm_reg **res) 3960 { 3961 struct umtx_shm_reg *reg, *reg1; 3962 struct ucred *cred; 3963 int error; 3964 3965 reg = umtx_shm_find_reg(key); 3966 if (reg != NULL) { 3967 *res = reg; 3968 return (0); 3969 } 3970 cred = td->td_ucred; 3971 if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP))) 3972 return (ENOMEM); 3973 reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO); 3974 reg->ushm_refcnt = 1; 3975 bcopy(key, ®->ushm_key, sizeof(*key)); 3976 reg->ushm_obj = shm_alloc(td->td_ucred, O_RDWR); 3977 reg->ushm_cred = crhold(cred); 3978 error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE); 3979 if (error != 0) { 3980 umtx_shm_free_reg(reg); 3981 return (error); 3982 } 3983 mtx_lock(&umtx_shm_lock); 3984 reg1 = umtx_shm_find_reg_locked(key); 3985 if (reg1 != NULL) { 3986 mtx_unlock(&umtx_shm_lock); 3987 umtx_shm_free_reg(reg); 3988 *res = reg1; 3989 return (0); 3990 } 3991 reg->ushm_refcnt++; 3992 TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link); 3993 LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg, 3994 ushm_obj_link); 3995 reg->ushm_flags = USHMF_REG_LINKED | USHMF_OBJ_LINKED; 3996 mtx_unlock(&umtx_shm_lock); 3997 *res = reg; 3998 return (0); 3999 } 4000 4001 static int 4002 umtx_shm_alive(struct thread *td, void *addr) 4003 { 4004 vm_map_t map; 4005 vm_map_entry_t entry; 4006 vm_object_t object; 4007 vm_pindex_t pindex; 4008 vm_prot_t prot; 4009 int res, ret; 4010 boolean_t wired; 4011 4012 map = &td->td_proc->p_vmspace->vm_map; 4013 res = vm_map_lookup(&map, (uintptr_t)addr, VM_PROT_READ, &entry, 4014 &object, &pindex, &prot, &wired); 4015 if (res != KERN_SUCCESS) 4016 return (EFAULT); 4017 if (object == NULL) 4018 ret = EINVAL; 4019 else 4020 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0; 4021 vm_map_lookup_done(map, entry); 4022 return (ret); 4023 } 4024 4025 static void 4026 umtx_shm_init(void) 4027 { 4028 int i; 4029 4030 umtx_shm_reg_zone = uma_zcreate("umtx_shm", sizeof(struct umtx_shm_reg), 4031 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 4032 mtx_init(&umtx_shm_lock, "umtxshm", NULL, MTX_DEF); 4033 for (i = 0; i < nitems(umtx_shm_registry); i++) 4034 TAILQ_INIT(&umtx_shm_registry[i]); 4035 } 4036 4037 static int 4038 umtx_shm(struct thread *td, void *addr, u_int flags) 4039 { 4040 struct umtx_key key; 4041 struct umtx_shm_reg *reg; 4042 struct file *fp; 4043 int error, fd; 4044 4045 if (__bitcount(flags & (UMTX_SHM_CREAT | UMTX_SHM_LOOKUP | 4046 UMTX_SHM_DESTROY| UMTX_SHM_ALIVE)) != 1) 4047 return (EINVAL); 4048 if ((flags & UMTX_SHM_ALIVE) != 0) 4049 return (umtx_shm_alive(td, addr)); 4050 error = umtx_key_get(addr, TYPE_SHM, PROCESS_SHARE, &key); 4051 if (error != 0) 4052 return (error); 4053 KASSERT(key.shared == 1, ("non-shared key")); 4054 if ((flags & UMTX_SHM_CREAT) != 0) { 4055 error = umtx_shm_create_reg(td, &key, ®); 4056 } else { 4057 reg = umtx_shm_find_reg(&key); 4058 if (reg == NULL) 4059 error = ESRCH; 4060 } 4061 umtx_key_release(&key); 4062 if (error != 0) 4063 return (error); 4064 KASSERT(reg != NULL, ("no reg")); 4065 if ((flags & UMTX_SHM_DESTROY) != 0) { 4066 umtx_shm_unref_reg(reg, true); 4067 } else { 4068 #if 0 4069 #ifdef MAC 4070 error = mac_posixshm_check_open(td->td_ucred, 4071 reg->ushm_obj, FFLAGS(O_RDWR)); 4072 if (error == 0) 4073 #endif 4074 error = shm_access(reg->ushm_obj, td->td_ucred, 4075 FFLAGS(O_RDWR)); 4076 if (error == 0) 4077 #endif 4078 error = falloc_caps(td, &fp, &fd, O_CLOEXEC, NULL); 4079 if (error == 0) { 4080 shm_hold(reg->ushm_obj); 4081 finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj, 4082 &shm_ops); 4083 td->td_retval[0] = fd; 4084 fdrop(fp, td); 4085 } 4086 } 4087 umtx_shm_unref_reg(reg, false); 4088 return (error); 4089 } 4090 4091 static int 4092 __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap) 4093 { 4094 4095 return (umtx_shm(td, uap->uaddr1, uap->val)); 4096 } 4097 4098 static int 4099 umtx_robust_lists(struct thread *td, struct umtx_robust_lists_params *rbp) 4100 { 4101 4102 td->td_rb_list = rbp->robust_list_offset; 4103 td->td_rbp_list = rbp->robust_priv_list_offset; 4104 td->td_rb_inact = rbp->robust_inact_offset; 4105 return (0); 4106 } 4107 4108 static int 4109 __umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap) 4110 { 4111 struct umtx_robust_lists_params rb; 4112 int error; 4113 4114 if (uap->val > sizeof(rb)) 4115 return (EINVAL); 4116 bzero(&rb, sizeof(rb)); 4117 error = copyin(uap->uaddr1, &rb, uap->val); 4118 if (error != 0) 4119 return (error); 4120 return (umtx_robust_lists(td, &rb)); 4121 } 4122 4123 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); 4124 4125 static const _umtx_op_func op_table[] = { 4126 [UMTX_OP_RESERVED0] = __umtx_op_unimpl, 4127 [UMTX_OP_RESERVED1] = __umtx_op_unimpl, 4128 [UMTX_OP_WAIT] = __umtx_op_wait, 4129 [UMTX_OP_WAKE] = __umtx_op_wake, 4130 [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, 4131 [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex, 4132 [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, 4133 [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, 4134 [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait, 4135 [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, 4136 [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast, 4137 [UMTX_OP_WAIT_UINT] = __umtx_op_wait_uint, 4138 [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock, 4139 [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock, 4140 [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock, 4141 [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private, 4142 [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private, 4143 [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex, 4144 [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex, 4145 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 4146 [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait, 4147 [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake, 4148 #else 4149 [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl, 4150 [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl, 4151 #endif 4152 [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private, 4153 [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex, 4154 [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait, 4155 [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, 4156 [UMTX_OP_SHM] = __umtx_op_shm, 4157 [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists, 4158 }; 4159 4160 int 4161 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap) 4162 { 4163 4164 if ((unsigned)uap->op < nitems(op_table)) 4165 return (*op_table[uap->op])(td, uap); 4166 return (EINVAL); 4167 } 4168 4169 #ifdef COMPAT_FREEBSD32 4170 4171 struct timespec32 { 4172 int32_t tv_sec; 4173 int32_t tv_nsec; 4174 }; 4175 4176 struct umtx_time32 { 4177 struct timespec32 timeout; 4178 uint32_t flags; 4179 uint32_t clockid; 4180 }; 4181 4182 static inline int 4183 umtx_copyin_timeout32(void *addr, struct timespec *tsp) 4184 { 4185 struct timespec32 ts32; 4186 int error; 4187 4188 error = copyin(addr, &ts32, sizeof(struct timespec32)); 4189 if (error == 0) { 4190 if (ts32.tv_sec < 0 || 4191 ts32.tv_nsec >= 1000000000 || 4192 ts32.tv_nsec < 0) 4193 error = EINVAL; 4194 else { 4195 tsp->tv_sec = ts32.tv_sec; 4196 tsp->tv_nsec = ts32.tv_nsec; 4197 } 4198 } 4199 return (error); 4200 } 4201 4202 static inline int 4203 umtx_copyin_umtx_time32(const void *addr, size_t size, struct _umtx_time *tp) 4204 { 4205 struct umtx_time32 t32; 4206 int error; 4207 4208 t32.clockid = CLOCK_REALTIME; 4209 t32.flags = 0; 4210 if (size <= sizeof(struct timespec32)) 4211 error = copyin(addr, &t32.timeout, sizeof(struct timespec32)); 4212 else 4213 error = copyin(addr, &t32, sizeof(struct umtx_time32)); 4214 if (error != 0) 4215 return (error); 4216 if (t32.timeout.tv_sec < 0 || 4217 t32.timeout.tv_nsec >= 1000000000 || t32.timeout.tv_nsec < 0) 4218 return (EINVAL); 4219 tp->_timeout.tv_sec = t32.timeout.tv_sec; 4220 tp->_timeout.tv_nsec = t32.timeout.tv_nsec; 4221 tp->_flags = t32.flags; 4222 tp->_clockid = t32.clockid; 4223 return (0); 4224 } 4225 4226 static int 4227 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4228 { 4229 struct _umtx_time *tm_p, timeout; 4230 int error; 4231 4232 if (uap->uaddr2 == NULL) 4233 tm_p = NULL; 4234 else { 4235 error = umtx_copyin_umtx_time32(uap->uaddr2, 4236 (size_t)uap->uaddr1, &timeout); 4237 if (error != 0) 4238 return (error); 4239 tm_p = &timeout; 4240 } 4241 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); 4242 } 4243 4244 static int 4245 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 4246 { 4247 struct _umtx_time *tm_p, timeout; 4248 int error; 4249 4250 /* Allow a null timespec (wait forever). */ 4251 if (uap->uaddr2 == NULL) 4252 tm_p = NULL; 4253 else { 4254 error = umtx_copyin_umtx_time32(uap->uaddr2, 4255 (size_t)uap->uaddr1, &timeout); 4256 if (error != 0) 4257 return (error); 4258 tm_p = &timeout; 4259 } 4260 return (do_lock_umutex(td, uap->obj, tm_p, 0)); 4261 } 4262 4263 static int 4264 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 4265 { 4266 struct _umtx_time *tm_p, timeout; 4267 int error; 4268 4269 /* Allow a null timespec (wait forever). */ 4270 if (uap->uaddr2 == NULL) 4271 tm_p = NULL; 4272 else { 4273 error = umtx_copyin_umtx_time32(uap->uaddr2, 4274 (size_t)uap->uaddr1, &timeout); 4275 if (error != 0) 4276 return (error); 4277 tm_p = &timeout; 4278 } 4279 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); 4280 } 4281 4282 static int 4283 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4284 { 4285 struct timespec *ts, timeout; 4286 int error; 4287 4288 /* Allow a null timespec (wait forever). */ 4289 if (uap->uaddr2 == NULL) 4290 ts = NULL; 4291 else { 4292 error = umtx_copyin_timeout32(uap->uaddr2, &timeout); 4293 if (error != 0) 4294 return (error); 4295 ts = &timeout; 4296 } 4297 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 4298 } 4299 4300 static int 4301 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap) 4302 { 4303 struct _umtx_time timeout; 4304 int error; 4305 4306 /* Allow a null timespec (wait forever). */ 4307 if (uap->uaddr2 == NULL) { 4308 error = do_rw_rdlock(td, uap->obj, uap->val, 0); 4309 } else { 4310 error = umtx_copyin_umtx_time32(uap->uaddr2, 4311 (size_t)uap->uaddr1, &timeout); 4312 if (error != 0) 4313 return (error); 4314 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout); 4315 } 4316 return (error); 4317 } 4318 4319 static int 4320 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap) 4321 { 4322 struct _umtx_time timeout; 4323 int error; 4324 4325 /* Allow a null timespec (wait forever). */ 4326 if (uap->uaddr2 == NULL) { 4327 error = do_rw_wrlock(td, uap->obj, 0); 4328 } else { 4329 error = umtx_copyin_umtx_time32(uap->uaddr2, 4330 (size_t)uap->uaddr1, &timeout); 4331 if (error != 0) 4332 return (error); 4333 error = do_rw_wrlock(td, uap->obj, &timeout); 4334 } 4335 return (error); 4336 } 4337 4338 static int 4339 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap) 4340 { 4341 struct _umtx_time *tm_p, timeout; 4342 int error; 4343 4344 if (uap->uaddr2 == NULL) 4345 tm_p = NULL; 4346 else { 4347 error = umtx_copyin_umtx_time32( 4348 uap->uaddr2, (size_t)uap->uaddr1,&timeout); 4349 if (error != 0) 4350 return (error); 4351 tm_p = &timeout; 4352 } 4353 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); 4354 } 4355 4356 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 4357 static int 4358 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4359 { 4360 struct _umtx_time *tm_p, timeout; 4361 int error; 4362 4363 /* Allow a null timespec (wait forever). */ 4364 if (uap->uaddr2 == NULL) 4365 tm_p = NULL; 4366 else { 4367 error = umtx_copyin_umtx_time32(uap->uaddr2, 4368 (size_t)uap->uaddr1, &timeout); 4369 if (error != 0) 4370 return (error); 4371 tm_p = &timeout; 4372 } 4373 return (do_sem_wait(td, uap->obj, tm_p)); 4374 } 4375 #endif 4376 4377 static int 4378 __umtx_op_sem2_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 4379 { 4380 struct _umtx_time *tm_p, timeout; 4381 size_t uasize; 4382 int error; 4383 4384 /* Allow a null timespec (wait forever). */ 4385 if (uap->uaddr2 == NULL) { 4386 uasize = 0; 4387 tm_p = NULL; 4388 } else { 4389 uasize = (size_t)uap->uaddr1; 4390 error = umtx_copyin_umtx_time32(uap->uaddr2, uasize, &timeout); 4391 if (error != 0) 4392 return (error); 4393 tm_p = &timeout; 4394 } 4395 error = do_sem2_wait(td, uap->obj, tm_p); 4396 if (error == EINTR && uap->uaddr2 != NULL && 4397 (timeout._flags & UMTX_ABSTIME) == 0 && 4398 uasize >= sizeof(struct umtx_time32) + sizeof(struct timespec32)) { 4399 struct timespec32 remain32 = { 4400 .tv_sec = timeout._timeout.tv_sec, 4401 .tv_nsec = timeout._timeout.tv_nsec 4402 }; 4403 error = copyout(&remain32, 4404 (struct umtx_time32 *)uap->uaddr2 + 1, 4405 sizeof(struct timespec32)); 4406 if (error == 0) { 4407 error = EINTR; 4408 } 4409 } 4410 4411 return (error); 4412 } 4413 4414 static int 4415 __umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap) 4416 { 4417 uint32_t uaddrs[BATCH_SIZE], **upp; 4418 int count, error, i, pos, tocopy; 4419 4420 upp = (uint32_t **)uap->obj; 4421 error = 0; 4422 for (count = uap->val, pos = 0; count > 0; count -= tocopy, 4423 pos += tocopy) { 4424 tocopy = MIN(count, BATCH_SIZE); 4425 error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t)); 4426 if (error != 0) 4427 break; 4428 for (i = 0; i < tocopy; ++i) 4429 kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i], 4430 INT_MAX, 1); 4431 maybe_yield(); 4432 } 4433 return (error); 4434 } 4435 4436 struct umtx_robust_lists_params_compat32 { 4437 uint32_t robust_list_offset; 4438 uint32_t robust_priv_list_offset; 4439 uint32_t robust_inact_offset; 4440 }; 4441 4442 static int 4443 __umtx_op_robust_lists_compat32(struct thread *td, struct _umtx_op_args *uap) 4444 { 4445 struct umtx_robust_lists_params rb; 4446 struct umtx_robust_lists_params_compat32 rb32; 4447 int error; 4448 4449 if (uap->val > sizeof(rb32)) 4450 return (EINVAL); 4451 bzero(&rb, sizeof(rb)); 4452 bzero(&rb32, sizeof(rb32)); 4453 error = copyin(uap->uaddr1, &rb32, uap->val); 4454 if (error != 0) 4455 return (error); 4456 rb.robust_list_offset = rb32.robust_list_offset; 4457 rb.robust_priv_list_offset = rb32.robust_priv_list_offset; 4458 rb.robust_inact_offset = rb32.robust_inact_offset; 4459 return (umtx_robust_lists(td, &rb)); 4460 } 4461 4462 static const _umtx_op_func op_table_compat32[] = { 4463 [UMTX_OP_RESERVED0] = __umtx_op_unimpl, 4464 [UMTX_OP_RESERVED1] = __umtx_op_unimpl, 4465 [UMTX_OP_WAIT] = __umtx_op_wait_compat32, 4466 [UMTX_OP_WAKE] = __umtx_op_wake, 4467 [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, 4468 [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex_compat32, 4469 [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, 4470 [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, 4471 [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait_compat32, 4472 [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, 4473 [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast, 4474 [UMTX_OP_WAIT_UINT] = __umtx_op_wait_compat32, 4475 [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock_compat32, 4476 [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock_compat32, 4477 [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock, 4478 [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private_compat32, 4479 [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private, 4480 [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex_compat32, 4481 [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex, 4482 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) 4483 [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait_compat32, 4484 [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake, 4485 #else 4486 [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl, 4487 [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl, 4488 #endif 4489 [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private32, 4490 [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex, 4491 [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait_compat32, 4492 [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, 4493 [UMTX_OP_SHM] = __umtx_op_shm, 4494 [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists_compat32, 4495 }; 4496 4497 int 4498 freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap) 4499 { 4500 4501 if ((unsigned)uap->op < nitems(op_table_compat32)) { 4502 return (*op_table_compat32[uap->op])(td, 4503 (struct _umtx_op_args *)uap); 4504 } 4505 return (EINVAL); 4506 } 4507 #endif 4508 4509 void 4510 umtx_thread_init(struct thread *td) 4511 { 4512 4513 td->td_umtxq = umtxq_alloc(); 4514 td->td_umtxq->uq_thread = td; 4515 } 4516 4517 void 4518 umtx_thread_fini(struct thread *td) 4519 { 4520 4521 umtxq_free(td->td_umtxq); 4522 } 4523 4524 /* 4525 * It will be called when new thread is created, e.g fork(). 4526 */ 4527 void 4528 umtx_thread_alloc(struct thread *td) 4529 { 4530 struct umtx_q *uq; 4531 4532 uq = td->td_umtxq; 4533 uq->uq_inherited_pri = PRI_MAX; 4534 4535 KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); 4536 KASSERT(uq->uq_thread == td, ("uq_thread != td")); 4537 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); 4538 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); 4539 } 4540 4541 /* 4542 * exec() hook. 4543 * 4544 * Clear robust lists for all process' threads, not delaying the 4545 * cleanup to thread_exit hook, since the relevant address space is 4546 * destroyed right now. 4547 */ 4548 static void 4549 umtx_exec_hook(void *arg __unused, struct proc *p, 4550 struct image_params *imgp __unused) 4551 { 4552 struct thread *td; 4553 4554 KASSERT(p == curproc, ("need curproc")); 4555 KASSERT((p->p_flag & P_HADTHREADS) == 0 || 4556 (p->p_flag & P_STOPPED_SINGLE) != 0, 4557 ("curproc must be single-threaded")); 4558 /* 4559 * There is no need to lock the list as only this thread can be 4560 * running. 4561 */ 4562 FOREACH_THREAD_IN_PROC(p, td) { 4563 KASSERT(td == curthread || 4564 ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)), 4565 ("running thread %p %p", p, td)); 4566 umtx_thread_cleanup(td); 4567 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0; 4568 } 4569 } 4570 4571 /* 4572 * thread_exit() hook. 4573 */ 4574 void 4575 umtx_thread_exit(struct thread *td) 4576 { 4577 4578 umtx_thread_cleanup(td); 4579 } 4580 4581 static int 4582 umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res) 4583 { 4584 u_long res1; 4585 #ifdef COMPAT_FREEBSD32 4586 uint32_t res32; 4587 #endif 4588 int error; 4589 4590 #ifdef COMPAT_FREEBSD32 4591 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { 4592 error = fueword32((void *)ptr, &res32); 4593 if (error == 0) 4594 res1 = res32; 4595 } else 4596 #endif 4597 { 4598 error = fueword((void *)ptr, &res1); 4599 } 4600 if (error == 0) 4601 *res = res1; 4602 else 4603 error = EFAULT; 4604 return (error); 4605 } 4606 4607 static void 4608 umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list) 4609 { 4610 #ifdef COMPAT_FREEBSD32 4611 struct umutex32 m32; 4612 4613 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { 4614 memcpy(&m32, m, sizeof(m32)); 4615 *rb_list = m32.m_rb_lnk; 4616 } else 4617 #endif 4618 *rb_list = m->m_rb_lnk; 4619 } 4620 4621 static int 4622 umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact) 4623 { 4624 struct umutex m; 4625 int error; 4626 4627 KASSERT(td->td_proc == curproc, ("need current vmspace")); 4628 error = copyin((void *)rbp, &m, sizeof(m)); 4629 if (error != 0) 4630 return (error); 4631 if (rb_list != NULL) 4632 umtx_read_rb_list(td, &m, rb_list); 4633 if ((m.m_flags & UMUTEX_ROBUST) == 0) 4634 return (EINVAL); 4635 if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid) 4636 /* inact is cleared after unlock, allow the inconsistency */ 4637 return (inact ? 0 : EINVAL); 4638 return (do_unlock_umutex(td, (struct umutex *)rbp, true)); 4639 } 4640 4641 static void 4642 umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact, 4643 const char *name) 4644 { 4645 int error, i; 4646 uintptr_t rbp; 4647 bool inact; 4648 4649 if (rb_list == 0) 4650 return; 4651 error = umtx_read_uptr(td, rb_list, &rbp); 4652 for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) { 4653 if (rbp == *rb_inact) { 4654 inact = true; 4655 *rb_inact = 0; 4656 } else 4657 inact = false; 4658 error = umtx_handle_rb(td, rbp, &rbp, inact); 4659 } 4660 if (i == umtx_max_rb && umtx_verbose_rb) { 4661 uprintf("comm %s pid %d: reached umtx %smax rb %d\n", 4662 td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb); 4663 } 4664 if (error != 0 && umtx_verbose_rb) { 4665 uprintf("comm %s pid %d: handling %srb error %d\n", 4666 td->td_proc->p_comm, td->td_proc->p_pid, name, error); 4667 } 4668 } 4669 4670 /* 4671 * Clean up umtx data. 4672 */ 4673 static void 4674 umtx_thread_cleanup(struct thread *td) 4675 { 4676 struct umtx_q *uq; 4677 struct umtx_pi *pi; 4678 uintptr_t rb_inact; 4679 4680 /* 4681 * Disown pi mutexes. 4682 */ 4683 uq = td->td_umtxq; 4684 if (uq != NULL) { 4685 if (uq->uq_inherited_pri != PRI_MAX || 4686 !TAILQ_EMPTY(&uq->uq_pi_contested)) { 4687 mtx_lock(&umtx_lock); 4688 uq->uq_inherited_pri = PRI_MAX; 4689 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { 4690 pi->pi_owner = NULL; 4691 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); 4692 } 4693 mtx_unlock(&umtx_lock); 4694 } 4695 sched_lend_user_prio_cond(td, PRI_MAX); 4696 } 4697 4698 if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0) 4699 return; 4700 4701 /* 4702 * Handle terminated robust mutexes. Must be done after 4703 * robust pi disown, otherwise unlock could see unowned 4704 * entries. 4705 */ 4706 rb_inact = td->td_rb_inact; 4707 if (rb_inact != 0) 4708 (void)umtx_read_uptr(td, rb_inact, &rb_inact); 4709 umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, ""); 4710 umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv "); 4711 if (rb_inact != 0) 4712 (void)umtx_handle_rb(td, rb_inact, NULL, true); 4713 } 4714