1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2009-2021 Dmitry Chagin <dchagin@FreeBSD.org> 5 * Copyright (c) 2008 Roman Divacky 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include "opt_compat.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/imgact.h> 37 #include <sys/imgact_elf.h> 38 #include <sys/ktr.h> 39 #include <sys/mutex.h> 40 #include <sys/priv.h> 41 #include <sys/proc.h> 42 #include <sys/sched.h> 43 #include <sys/umtxvar.h> 44 45 #ifdef COMPAT_LINUX32 46 #include <machine/../linux32/linux.h> 47 #include <machine/../linux32/linux32_proto.h> 48 #else 49 #include <machine/../linux/linux.h> 50 #include <machine/../linux/linux_proto.h> 51 #endif 52 #include <compat/linux/linux_emul.h> 53 #include <compat/linux/linux_futex.h> 54 #include <compat/linux/linux_misc.h> 55 #include <compat/linux/linux_timer.h> 56 #include <compat/linux/linux_util.h> 57 58 #define FUTEX_SHARED 0x8 /* shared futex */ 59 60 #define GET_SHARED(a) (a->flags & FUTEX_SHARED) ? AUTO_SHARE : THREAD_SHARE 61 62 static int futex_atomic_op(struct thread *, int, uint32_t *, int *); 63 static int handle_futex_death(struct thread *td, struct linux_emuldata *, 64 uint32_t *, unsigned int, bool); 65 static int fetch_robust_entry(struct linux_robust_list **, 66 struct linux_robust_list **, unsigned int *); 67 68 struct linux_futex_args { 69 uint32_t *uaddr; 70 int32_t op; 71 uint32_t flags; 72 bool clockrt; 73 uint32_t val; 74 struct timespec *ts; 75 uint32_t *uaddr2; 76 uint32_t val3; 77 bool val3_compare; 78 struct timespec kts; 79 }; 80 81 static inline int futex_key_get(const void *, int, int, struct umtx_key *); 82 static void linux_umtx_abs_timeout_init(struct umtx_abs_timeout *, 83 struct linux_futex_args *); 84 static int linux_futex(struct thread *, struct linux_futex_args *); 85 static int linux_futex_wait(struct thread *, struct linux_futex_args *); 86 static int linux_futex_wake(struct thread *, struct linux_futex_args *); 87 static int linux_futex_requeue(struct thread *, struct linux_futex_args *); 88 static int linux_futex_wakeop(struct thread *, struct linux_futex_args *); 89 static int linux_futex_lock_pi(struct thread *, bool, struct linux_futex_args *); 90 static int linux_futex_unlock_pi(struct thread *, bool, 91 struct linux_futex_args *); 92 static int futex_wake_pi(struct thread *, uint32_t *, bool); 93 94 static int 95 futex_key_get(const void *uaddr, int type, int share, struct umtx_key *key) 96 { 97 98 /* Check that futex address is a 32bit aligned. */ 99 if (!__is_aligned(uaddr, sizeof(uint32_t))) 100 return (EINVAL); 101 return (umtx_key_get(uaddr, type, share, key)); 102 } 103 104 int 105 futex_wake(struct thread *td, uint32_t *uaddr, int val, bool shared) 106 { 107 struct linux_futex_args args; 108 109 bzero(&args, sizeof(args)); 110 args.op = LINUX_FUTEX_WAKE; 111 args.uaddr = uaddr; 112 args.flags = shared == true ? FUTEX_SHARED : 0; 113 args.val = val; 114 args.val3 = FUTEX_BITSET_MATCH_ANY; 115 116 return (linux_futex_wake(td, &args)); 117 } 118 119 static int 120 futex_wake_pi(struct thread *td, uint32_t *uaddr, bool shared) 121 { 122 struct linux_futex_args args; 123 124 bzero(&args, sizeof(args)); 125 args.op = LINUX_FUTEX_UNLOCK_PI; 126 args.uaddr = uaddr; 127 args.flags = shared == true ? FUTEX_SHARED : 0; 128 129 return (linux_futex_unlock_pi(td, true, &args)); 130 } 131 132 static int 133 futex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr, 134 int *res) 135 { 136 int op = (encoded_op >> 28) & 7; 137 int cmp = (encoded_op >> 24) & 15; 138 int oparg = (encoded_op << 8) >> 20; 139 int cmparg = (encoded_op << 20) >> 20; 140 int oldval = 0, ret; 141 142 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) 143 oparg = 1 << oparg; 144 145 switch (op) { 146 case FUTEX_OP_SET: 147 ret = futex_xchgl(oparg, uaddr, &oldval); 148 break; 149 case FUTEX_OP_ADD: 150 ret = futex_addl(oparg, uaddr, &oldval); 151 break; 152 case FUTEX_OP_OR: 153 ret = futex_orl(oparg, uaddr, &oldval); 154 break; 155 case FUTEX_OP_ANDN: 156 ret = futex_andl(~oparg, uaddr, &oldval); 157 break; 158 case FUTEX_OP_XOR: 159 ret = futex_xorl(oparg, uaddr, &oldval); 160 break; 161 default: 162 ret = ENOSYS; 163 break; 164 } 165 166 if (ret != 0) 167 return (ret); 168 169 switch (cmp) { 170 case FUTEX_OP_CMP_EQ: 171 *res = (oldval == cmparg); 172 break; 173 case FUTEX_OP_CMP_NE: 174 *res = (oldval != cmparg); 175 break; 176 case FUTEX_OP_CMP_LT: 177 *res = (oldval < cmparg); 178 break; 179 case FUTEX_OP_CMP_GE: 180 *res = (oldval >= cmparg); 181 break; 182 case FUTEX_OP_CMP_LE: 183 *res = (oldval <= cmparg); 184 break; 185 case FUTEX_OP_CMP_GT: 186 *res = (oldval > cmparg); 187 break; 188 default: 189 ret = ENOSYS; 190 } 191 192 return (ret); 193 } 194 195 static int 196 linux_futex(struct thread *td, struct linux_futex_args *args) 197 { 198 struct linux_pemuldata *pem; 199 struct proc *p; 200 201 if (args->op & LINUX_FUTEX_PRIVATE_FLAG) { 202 args->flags = 0; 203 args->op &= ~LINUX_FUTEX_PRIVATE_FLAG; 204 } else 205 args->flags = FUTEX_SHARED; 206 207 args->clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME; 208 args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME; 209 210 if (args->clockrt && 211 args->op != LINUX_FUTEX_WAIT_BITSET && 212 args->op != LINUX_FUTEX_WAIT_REQUEUE_PI && 213 args->op != LINUX_FUTEX_LOCK_PI2) 214 return (ENOSYS); 215 216 switch (args->op) { 217 case LINUX_FUTEX_WAIT: 218 args->val3 = FUTEX_BITSET_MATCH_ANY; 219 /* FALLTHROUGH */ 220 221 case LINUX_FUTEX_WAIT_BITSET: 222 LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x", 223 args->uaddr, args->val, args->val3); 224 225 return (linux_futex_wait(td, args)); 226 227 case LINUX_FUTEX_WAKE: 228 args->val3 = FUTEX_BITSET_MATCH_ANY; 229 /* FALLTHROUGH */ 230 231 case LINUX_FUTEX_WAKE_BITSET: 232 LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x", 233 args->uaddr, args->val, args->val3); 234 235 return (linux_futex_wake(td, args)); 236 237 case LINUX_FUTEX_REQUEUE: 238 /* 239 * Glibc does not use this operation since version 2.3.3, 240 * as it is racy and replaced by FUTEX_CMP_REQUEUE operation. 241 * Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when 242 * FUTEX_REQUEUE returned EINVAL. 243 */ 244 pem = pem_find(td->td_proc); 245 if ((pem->flags & LINUX_XDEPR_REQUEUEOP) == 0) { 246 linux_msg(td, "unsupported FUTEX_REQUEUE"); 247 pem->flags |= LINUX_XDEPR_REQUEUEOP; 248 } 249 250 /* 251 * The above is true, however musl libc does make use of the 252 * futex requeue operation, allow operation for brands which 253 * set LINUX_BI_FUTEX_REQUEUE bit of Brandinfo flags. 254 */ 255 p = td->td_proc; 256 Elf_Brandinfo *bi = p->p_elf_brandinfo; 257 if (bi == NULL || ((bi->flags & LINUX_BI_FUTEX_REQUEUE)) == 0) 258 return (EINVAL); 259 args->val3_compare = false; 260 /* FALLTHROUGH */ 261 262 case LINUX_FUTEX_CMP_REQUEUE: 263 LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p " 264 "nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x", 265 args->uaddr, args->val, args->val3, args->uaddr2, 266 args->ts); 267 268 return (linux_futex_requeue(td, args)); 269 270 case LINUX_FUTEX_WAKE_OP: 271 LINUX_CTR5(sys_futex, "WAKE_OP " 272 "uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x", 273 args->uaddr, args->val, args->uaddr2, args->val3, 274 args->ts); 275 276 return (linux_futex_wakeop(td, args)); 277 278 case LINUX_FUTEX_LOCK_PI: 279 args->clockrt = true; 280 /* FALLTHROUGH */ 281 282 case LINUX_FUTEX_LOCK_PI2: 283 LINUX_CTR2(sys_futex, "LOCKPI uaddr %p val 0x%x", 284 args->uaddr, args->val); 285 286 return (linux_futex_lock_pi(td, false, args)); 287 288 case LINUX_FUTEX_UNLOCK_PI: 289 LINUX_CTR1(sys_futex, "UNLOCKPI uaddr %p", 290 args->uaddr); 291 292 return (linux_futex_unlock_pi(td, false, args)); 293 294 case LINUX_FUTEX_TRYLOCK_PI: 295 LINUX_CTR1(sys_futex, "TRYLOCKPI uaddr %p", 296 args->uaddr); 297 298 return (linux_futex_lock_pi(td, true, args)); 299 300 /* 301 * Current implementation of FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI 302 * can't be used anymore to implement conditional variables. 303 * A detailed explanation can be found here: 304 * 305 * https://sourceware.org/bugzilla/show_bug.cgi?id=13165 306 * and here http://austingroupbugs.net/view.php?id=609 307 * 308 * And since commit 309 * https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=ed19993b5b0d05d62cc883571519a67dae481a14 310 * glibc does not use them. 311 */ 312 case LINUX_FUTEX_WAIT_REQUEUE_PI: 313 /* not yet implemented */ 314 pem = pem_find(td->td_proc); 315 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) { 316 linux_msg(td, "unsupported FUTEX_WAIT_REQUEUE_PI"); 317 pem->flags |= LINUX_XUNSUP_FUTEXPIOP; 318 } 319 return (ENOSYS); 320 321 case LINUX_FUTEX_CMP_REQUEUE_PI: 322 /* not yet implemented */ 323 pem = pem_find(td->td_proc); 324 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) { 325 linux_msg(td, "unsupported FUTEX_CMP_REQUEUE_PI"); 326 pem->flags |= LINUX_XUNSUP_FUTEXPIOP; 327 } 328 return (ENOSYS); 329 330 default: 331 linux_msg(td, "unsupported futex op %d", args->op); 332 return (ENOSYS); 333 } 334 } 335 336 /* 337 * pi protocol: 338 * - 0 futex word value means unlocked. 339 * - TID futex word value means locked. 340 * Userspace uses atomic ops to lock/unlock these futexes without entering the 341 * kernel. If the lock-acquire fastpath fails, (transition from 0 to TID fails), 342 * then FUTEX_LOCK_PI is called. 343 * The kernel atomically set FUTEX_WAITERS bit in the futex word value, if no 344 * other waiters exists looks up the thread that owns the futex (it has put its 345 * own TID into the futex value) and made this thread the owner of the internal 346 * pi-aware lock object (mutex). Then the kernel tries to lock the internal lock 347 * object, on which it blocks. Once it returns, it has the mutex acquired, and it 348 * sets the futex value to its own TID and returns (futex value contains 349 * FUTEX_WAITERS|TID). 350 * The unlock fastpath would fail (because the FUTEX_WAITERS bit is set) and 351 * FUTEX_UNLOCK_PI will be called. 352 * If a futex is found to be held at exit time, the kernel sets the OWNER_DIED 353 * bit of the futex word and wakes up the next futex waiter (if any), WAITERS 354 * bit is preserved (if any). 355 * If OWNER_DIED bit is set the kernel sanity checks the futex word value against 356 * the internal futex state and if correct, acquire futex. 357 */ 358 static int 359 linux_futex_lock_pi(struct thread *td, bool try, struct linux_futex_args *args) 360 { 361 struct umtx_abs_timeout timo; 362 struct linux_emuldata *em; 363 struct umtx_pi *pi, *new_pi; 364 struct thread *td1; 365 struct umtx_q *uq; 366 int error, rv; 367 uint32_t owner, old_owner; 368 369 em = em_find(td); 370 uq = td->td_umtxq; 371 error = futex_key_get(args->uaddr, TYPE_PI_FUTEX, GET_SHARED(args), 372 &uq->uq_key); 373 if (error != 0) 374 return (error); 375 if (args->ts != NULL) 376 linux_umtx_abs_timeout_init(&timo, args); 377 378 umtxq_lock(&uq->uq_key); 379 pi = umtx_pi_lookup(&uq->uq_key); 380 if (pi == NULL) { 381 new_pi = umtx_pi_alloc(M_NOWAIT); 382 if (new_pi == NULL) { 383 umtxq_unlock(&uq->uq_key); 384 new_pi = umtx_pi_alloc(M_WAITOK); 385 umtxq_lock(&uq->uq_key); 386 pi = umtx_pi_lookup(&uq->uq_key); 387 if (pi != NULL) { 388 umtx_pi_free(new_pi); 389 new_pi = NULL; 390 } 391 } 392 if (new_pi != NULL) { 393 new_pi->pi_key = uq->uq_key; 394 umtx_pi_insert(new_pi); 395 pi = new_pi; 396 } 397 } 398 umtx_pi_ref(pi); 399 umtxq_unlock(&uq->uq_key); 400 for (;;) { 401 /* Try uncontested case first. */ 402 rv = casueword32(args->uaddr, 0, &owner, em->em_tid); 403 /* The acquire succeeded. */ 404 if (rv == 0) { 405 error = 0; 406 break; 407 } 408 if (rv == -1) { 409 error = EFAULT; 410 break; 411 } 412 413 /* 414 * Avoid overwriting a possible error from sleep due 415 * to the pending signal with suspension check result. 416 */ 417 if (error == 0) { 418 error = thread_check_susp(td, true); 419 if (error != 0) 420 break; 421 } 422 423 /* The futex word at *uaddr is already locked by the caller. */ 424 if ((owner & FUTEX_TID_MASK) == em->em_tid) { 425 error = EDEADLK; 426 break; 427 } 428 429 /* 430 * Futex owner died, handle_futex_death() set the OWNER_DIED bit 431 * and clear tid. Try to acquire it. 432 */ 433 if ((owner & FUTEX_TID_MASK) == 0) { 434 old_owner = owner; 435 owner = owner & (FUTEX_WAITERS | FUTEX_OWNER_DIED); 436 owner |= em->em_tid; 437 rv = casueword32(args->uaddr, old_owner, &owner, owner); 438 if (rv == -1) { 439 error = EFAULT; 440 break; 441 } 442 if (rv == 1) { 443 if (error == 0) { 444 error = thread_check_susp(td, true); 445 if (error != 0) 446 break; 447 } 448 449 /* 450 * If this failed the lock could 451 * changed, restart. 452 */ 453 continue; 454 } 455 456 umtxq_lock(&uq->uq_key); 457 umtxq_busy(&uq->uq_key); 458 error = umtx_pi_claim(pi, td); 459 umtxq_unbusy(&uq->uq_key); 460 umtxq_unlock(&uq->uq_key); 461 if (error != 0) { 462 /* 463 * Since we're going to return an 464 * error, restore the futex to its 465 * previous, unowned state to avoid 466 * compounding the problem. 467 */ 468 (void)casuword32(args->uaddr, owner, old_owner); 469 } 470 break; 471 } 472 473 /* 474 * Inconsistent state: OWNER_DIED is set and tid is not 0. 475 * Linux does some checks of futex state, we return EINVAL, 476 * as the user space can take care of this. 477 */ 478 if ((owner & FUTEX_OWNER_DIED) != 0) { 479 error = EINVAL; 480 break; 481 } 482 483 if (try != 0) { 484 error = EBUSY; 485 break; 486 } 487 488 /* 489 * If we caught a signal, we have retried and now 490 * exit immediately. 491 */ 492 if (error != 0) 493 break; 494 495 umtxq_lock(&uq->uq_key); 496 umtxq_busy(&uq->uq_key); 497 umtxq_unlock(&uq->uq_key); 498 499 /* 500 * Set the contested bit so that a release in user space knows 501 * to use the system call for unlock. If this fails either some 502 * one else has acquired the lock or it has been released. 503 */ 504 rv = casueword32(args->uaddr, owner, &owner, 505 owner | FUTEX_WAITERS); 506 if (rv == -1) { 507 umtxq_unbusy_unlocked(&uq->uq_key); 508 error = EFAULT; 509 break; 510 } 511 if (rv == 1) { 512 umtxq_unbusy_unlocked(&uq->uq_key); 513 error = thread_check_susp(td, true); 514 if (error != 0) 515 break; 516 517 /* 518 * The lock changed and we need to retry or we 519 * lost a race to the thread unlocking the umtx. 520 */ 521 continue; 522 } 523 524 /* 525 * Substitute Linux thread id by native thread id to 526 * avoid refactoring code of umtxq_sleep_pi(). 527 */ 528 td1 = linux_tdfind(td, owner & FUTEX_TID_MASK, -1); 529 if (td1 != NULL) { 530 owner = td1->td_tid; 531 PROC_UNLOCK(td1->td_proc); 532 } else { 533 umtxq_unbusy_unlocked(&uq->uq_key); 534 error = EINVAL; 535 break; 536 } 537 538 umtxq_lock(&uq->uq_key); 539 540 /* We set the contested bit, sleep. */ 541 error = umtxq_sleep_pi(uq, pi, owner, "futexp", 542 args->ts == NULL ? NULL : &timo, 543 (args->flags & FUTEX_SHARED) != 0); 544 if (error != 0) 545 continue; 546 547 error = thread_check_susp(td, false); 548 if (error != 0) 549 break; 550 } 551 552 umtxq_lock(&uq->uq_key); 553 umtx_pi_unref(pi); 554 umtxq_unlock(&uq->uq_key); 555 umtx_key_release(&uq->uq_key); 556 return (error); 557 } 558 559 static int 560 linux_futex_unlock_pi(struct thread *td, bool rb, struct linux_futex_args *args) 561 { 562 struct linux_emuldata *em; 563 struct umtx_key key; 564 uint32_t old, owner, new_owner; 565 int count, error; 566 567 em = em_find(td); 568 569 /* 570 * Make sure we own this mtx. 571 */ 572 error = fueword32(args->uaddr, &owner); 573 if (error == -1) 574 return (EFAULT); 575 if (!rb && (owner & FUTEX_TID_MASK) != em->em_tid) 576 return (EPERM); 577 578 error = futex_key_get(args->uaddr, TYPE_PI_FUTEX, GET_SHARED(args), &key); 579 if (error != 0) 580 return (error); 581 umtxq_lock(&key); 582 umtxq_busy(&key); 583 error = umtx_pi_drop(td, &key, rb, &count); 584 if (error != 0 || rb) { 585 umtxq_unbusy(&key); 586 umtxq_unlock(&key); 587 umtx_key_release(&key); 588 return (error); 589 } 590 umtxq_unlock(&key); 591 592 /* 593 * When unlocking the futex, it must be marked as unowned if 594 * there is zero or one thread only waiting for it. 595 * Otherwise, it must be marked as contested. 596 */ 597 if (count > 1) 598 new_owner = FUTEX_WAITERS; 599 else 600 new_owner = 0; 601 602 again: 603 error = casueword32(args->uaddr, owner, &old, new_owner); 604 if (error == 1) { 605 error = thread_check_susp(td, false); 606 if (error == 0) 607 goto again; 608 } 609 umtxq_unbusy_unlocked(&key); 610 umtx_key_release(&key); 611 if (error == -1) 612 return (EFAULT); 613 if (error == 0 && old != owner) 614 return (EINVAL); 615 return (error); 616 } 617 618 static int 619 linux_futex_wakeop(struct thread *td, struct linux_futex_args *args) 620 { 621 struct umtx_key key, key2; 622 int nrwake, op_ret, ret; 623 int error, count; 624 625 if (args->uaddr == args->uaddr2) 626 return (EINVAL); 627 628 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args), &key); 629 if (error != 0) 630 return (error); 631 error = futex_key_get(args->uaddr2, TYPE_FUTEX, GET_SHARED(args), &key2); 632 if (error != 0) { 633 umtx_key_release(&key); 634 return (error); 635 } 636 umtxq_lock(&key); 637 umtxq_busy(&key); 638 umtxq_unlock(&key); 639 error = futex_atomic_op(td, args->val3, args->uaddr2, &op_ret); 640 umtxq_lock(&key); 641 umtxq_unbusy(&key); 642 if (error != 0) 643 goto out; 644 ret = umtxq_signal_mask(&key, args->val, args->val3); 645 if (op_ret > 0) { 646 nrwake = (int)(unsigned long)args->ts; 647 umtxq_lock(&key2); 648 count = umtxq_count(&key2); 649 if (count > 0) 650 ret += umtxq_signal_mask(&key2, nrwake, args->val3); 651 else 652 ret += umtxq_signal_mask(&key, nrwake, args->val3); 653 umtxq_unlock(&key2); 654 } 655 td->td_retval[0] = ret; 656 out: 657 umtxq_unlock(&key); 658 umtx_key_release(&key2); 659 umtx_key_release(&key); 660 return (error); 661 } 662 663 static int 664 linux_futex_requeue(struct thread *td, struct linux_futex_args *args) 665 { 666 int nrwake, nrrequeue; 667 struct umtx_key key, key2; 668 int error; 669 uint32_t uval; 670 671 /* 672 * Linux allows this, we would not, it is an incorrect 673 * usage of declared ABI, so return EINVAL. 674 */ 675 if (args->uaddr == args->uaddr2) 676 return (EINVAL); 677 678 nrrequeue = (int)(unsigned long)args->ts; 679 nrwake = args->val; 680 /* 681 * Sanity check to prevent signed integer overflow, 682 * see Linux CVE-2018-6927 683 */ 684 if (nrwake < 0 || nrrequeue < 0) 685 return (EINVAL); 686 687 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args), &key); 688 if (error != 0) 689 return (error); 690 error = futex_key_get(args->uaddr2, TYPE_FUTEX, GET_SHARED(args), &key2); 691 if (error != 0) { 692 umtx_key_release(&key); 693 return (error); 694 } 695 umtxq_lock(&key); 696 umtxq_busy(&key); 697 umtxq_unlock(&key); 698 error = fueword32(args->uaddr, &uval); 699 if (error != 0) 700 error = EFAULT; 701 else if (args->val3_compare == true && uval != args->val3) 702 error = EWOULDBLOCK; 703 umtxq_lock(&key); 704 umtxq_unbusy(&key); 705 if (error == 0) { 706 umtxq_lock(&key2); 707 td->td_retval[0] = umtxq_requeue(&key, nrwake, &key2, nrrequeue); 708 umtxq_unlock(&key2); 709 } 710 umtxq_unlock(&key); 711 umtx_key_release(&key2); 712 umtx_key_release(&key); 713 return (error); 714 } 715 716 static int 717 linux_futex_wake(struct thread *td, struct linux_futex_args *args) 718 { 719 struct umtx_key key; 720 int error; 721 722 if (args->val3 == 0) 723 return (EINVAL); 724 725 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args), &key); 726 if (error != 0) 727 return (error); 728 umtxq_lock(&key); 729 td->td_retval[0] = umtxq_signal_mask(&key, args->val, args->val3); 730 umtxq_unlock(&key); 731 umtx_key_release(&key); 732 return (0); 733 } 734 735 static int 736 linux_futex_wait(struct thread *td, struct linux_futex_args *args) 737 { 738 struct umtx_abs_timeout timo; 739 struct umtx_q *uq; 740 uint32_t uval; 741 int error; 742 743 if (args->val3 == 0) 744 error = EINVAL; 745 746 uq = td->td_umtxq; 747 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args), 748 &uq->uq_key); 749 if (error != 0) 750 return (error); 751 if (args->ts != NULL) 752 linux_umtx_abs_timeout_init(&timo, args); 753 umtxq_lock(&uq->uq_key); 754 umtxq_busy(&uq->uq_key); 755 uq->uq_bitset = args->val3; 756 umtxq_insert(uq); 757 umtxq_unlock(&uq->uq_key); 758 error = fueword32(args->uaddr, &uval); 759 if (error != 0) 760 error = EFAULT; 761 else if (uval != args->val) 762 error = EWOULDBLOCK; 763 umtxq_lock(&uq->uq_key); 764 umtxq_unbusy(&uq->uq_key); 765 if (error == 0) { 766 error = umtxq_sleep(uq, "futex", 767 args->ts == NULL ? NULL : &timo); 768 if ((uq->uq_flags & UQF_UMTXQ) == 0) 769 error = 0; 770 else 771 umtxq_remove(uq); 772 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) { 773 umtxq_remove(uq); 774 } 775 umtxq_unlock(&uq->uq_key); 776 umtx_key_release(&uq->uq_key); 777 return (error); 778 } 779 780 static void 781 linux_umtx_abs_timeout_init(struct umtx_abs_timeout *timo, 782 struct linux_futex_args *args) 783 { 784 int clockid, absolute; 785 786 /* 787 * The FUTEX_CLOCK_REALTIME option bit can be employed only with the 788 * FUTEX_WAIT_BITSET, FUTEX_WAIT_REQUEUE_PI, FUTEX_LOCK_PI2. 789 * For FUTEX_WAIT, timeout is interpreted as a relative value, for other 790 * futex operations timeout is interpreted as an absolute value. 791 * If FUTEX_CLOCK_REALTIME option bit is set, the Linux kernel measures 792 * the timeout against the CLOCK_REALTIME clock, otherwise the kernel 793 * measures the timeout against the CLOCK_MONOTONIC clock. 794 */ 795 clockid = args->clockrt ? CLOCK_REALTIME : CLOCK_MONOTONIC; 796 absolute = args->op == LINUX_FUTEX_WAIT ? false : true; 797 umtx_abs_timeout_init(timo, clockid, absolute, args->ts); 798 } 799 800 int 801 linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args) 802 { 803 struct linux_futex_args fargs = { 804 .uaddr = args->uaddr, 805 .op = args->op, 806 .val = args->val, 807 .ts = NULL, 808 .uaddr2 = args->uaddr2, 809 .val3 = args->val3, 810 .val3_compare = true, 811 }; 812 int error; 813 814 switch (args->op & LINUX_FUTEX_CMD_MASK) { 815 case LINUX_FUTEX_WAIT: 816 case LINUX_FUTEX_WAIT_BITSET: 817 case LINUX_FUTEX_LOCK_PI: 818 case LINUX_FUTEX_LOCK_PI2: 819 if (args->timeout != NULL) { 820 error = linux_get_timespec(&fargs.kts, args->timeout); 821 if (error != 0) 822 return (error); 823 fargs.ts = &fargs.kts; 824 } 825 break; 826 default: 827 fargs.ts = PTRIN(args->timeout); 828 } 829 return (linux_futex(td, &fargs)); 830 } 831 832 #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) 833 int 834 linux_sys_futex_time64(struct thread *td, 835 struct linux_sys_futex_time64_args *args) 836 { 837 struct linux_futex_args fargs = { 838 .uaddr = args->uaddr, 839 .op = args->op, 840 .val = args->val, 841 .ts = NULL, 842 .uaddr2 = args->uaddr2, 843 .val3 = args->val3, 844 .val3_compare = true, 845 }; 846 int error; 847 848 switch (args->op & LINUX_FUTEX_CMD_MASK) { 849 case LINUX_FUTEX_WAIT: 850 case LINUX_FUTEX_WAIT_BITSET: 851 case LINUX_FUTEX_LOCK_PI: 852 case LINUX_FUTEX_LOCK_PI2: 853 if (args->timeout != NULL) { 854 error = linux_get_timespec64(&fargs.kts, args->timeout); 855 if (error != 0) 856 return (error); 857 fargs.ts = &fargs.kts; 858 } 859 break; 860 default: 861 fargs.ts = PTRIN(args->timeout); 862 } 863 return (linux_futex(td, &fargs)); 864 } 865 #endif 866 867 int 868 linux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args) 869 { 870 struct linux_emuldata *em; 871 872 if (args->len != sizeof(struct linux_robust_list_head)) 873 return (EINVAL); 874 875 em = em_find(td); 876 em->robust_futexes = args->head; 877 878 return (0); 879 } 880 881 int 882 linux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args) 883 { 884 struct linux_emuldata *em; 885 struct linux_robust_list_head *head; 886 l_size_t len; 887 struct thread *td2; 888 int error; 889 890 if (!args->pid) { 891 em = em_find(td); 892 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n")); 893 head = em->robust_futexes; 894 } else { 895 td2 = linux_tdfind(td, args->pid, -1); 896 if (td2 == NULL) 897 return (ESRCH); 898 if (SV_PROC_ABI(td2->td_proc) != SV_ABI_LINUX) { 899 PROC_UNLOCK(td2->td_proc); 900 return (EPERM); 901 } 902 903 em = em_find(td2); 904 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n")); 905 /* XXX: ptrace? */ 906 if (priv_check(td, PRIV_CRED_SETUID) || 907 priv_check(td, PRIV_CRED_SETEUID) || 908 p_candebug(td, td2->td_proc)) { 909 PROC_UNLOCK(td2->td_proc); 910 return (EPERM); 911 } 912 head = em->robust_futexes; 913 914 PROC_UNLOCK(td2->td_proc); 915 } 916 917 len = sizeof(struct linux_robust_list_head); 918 error = copyout(&len, args->len, sizeof(l_size_t)); 919 if (error != 0) 920 return (EFAULT); 921 922 return (copyout(&head, args->head, sizeof(head))); 923 } 924 925 static int 926 handle_futex_death(struct thread *td, struct linux_emuldata *em, uint32_t *uaddr, 927 unsigned int pi, bool pending_op) 928 { 929 uint32_t uval, nval, mval; 930 int error; 931 932 retry: 933 error = fueword32(uaddr, &uval); 934 if (error != 0) 935 return (EFAULT); 936 937 /* 938 * Special case for regular (non PI) futexes. The unlock path in 939 * user space has two race scenarios: 940 * 941 * 1. The unlock path releases the user space futex value and 942 * before it can execute the futex() syscall to wake up 943 * waiters it is killed. 944 * 945 * 2. A woken up waiter is killed before it can acquire the 946 * futex in user space. 947 * 948 * In both cases the TID validation below prevents a wakeup of 949 * potential waiters which can cause these waiters to block 950 * forever. 951 * 952 * In both cases it is safe to attempt waking up a potential 953 * waiter without touching the user space futex value and trying 954 * to set the OWNER_DIED bit. 955 */ 956 if (pending_op && !pi && !uval) { 957 (void)futex_wake(td, uaddr, 1, true); 958 return (0); 959 } 960 961 if ((uval & FUTEX_TID_MASK) == em->em_tid) { 962 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED; 963 error = casueword32(uaddr, uval, &nval, mval); 964 if (error == -1) 965 return (EFAULT); 966 if (error == 1) { 967 error = thread_check_susp(td, false); 968 if (error != 0) 969 return (error); 970 goto retry; 971 } 972 973 if (!pi && (uval & FUTEX_WAITERS)) { 974 error = futex_wake(td, uaddr, 1, true); 975 if (error != 0) 976 return (error); 977 } else if (pi && (uval & FUTEX_WAITERS)) { 978 error = futex_wake_pi(td, uaddr, true); 979 if (error != 0) 980 return (error); 981 } 982 } 983 984 return (0); 985 } 986 987 static int 988 fetch_robust_entry(struct linux_robust_list **entry, 989 struct linux_robust_list **head, unsigned int *pi) 990 { 991 l_ulong uentry; 992 int error; 993 994 error = copyin((const void *)head, &uentry, sizeof(uentry)); 995 if (error != 0) 996 return (EFAULT); 997 998 *entry = (void *)(uentry & ~1UL); 999 *pi = uentry & 1; 1000 1001 return (0); 1002 } 1003 1004 #define LINUX_HANDLE_DEATH_PENDING true 1005 #define LINUX_HANDLE_DEATH_LIST false 1006 1007 /* This walks the list of robust futexes releasing them. */ 1008 void 1009 release_futexes(struct thread *td, struct linux_emuldata *em) 1010 { 1011 struct linux_robust_list_head *head; 1012 struct linux_robust_list *entry, *next_entry, *pending; 1013 unsigned int limit = 2048, pi, next_pi, pip; 1014 uint32_t *uaddr; 1015 l_long futex_offset; 1016 int error; 1017 1018 head = em->robust_futexes; 1019 if (head == NULL) 1020 return; 1021 1022 if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi)) 1023 return; 1024 1025 error = copyin(&head->futex_offset, &futex_offset, 1026 sizeof(futex_offset)); 1027 if (error != 0) 1028 return; 1029 1030 if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip)) 1031 return; 1032 1033 while (entry != &head->list) { 1034 error = fetch_robust_entry(&next_entry, PTRIN(&entry->next), 1035 &next_pi); 1036 1037 /* 1038 * A pending lock might already be on the list, so 1039 * don't process it twice. 1040 */ 1041 if (entry != pending) { 1042 uaddr = (uint32_t *)((caddr_t)entry + futex_offset); 1043 if (handle_futex_death(td, em, uaddr, pi, 1044 LINUX_HANDLE_DEATH_LIST)) 1045 return; 1046 } 1047 if (error != 0) 1048 return; 1049 1050 entry = next_entry; 1051 pi = next_pi; 1052 1053 if (!--limit) 1054 break; 1055 1056 sched_relinquish(curthread); 1057 } 1058 1059 if (pending) { 1060 uaddr = (uint32_t *)((caddr_t)pending + futex_offset); 1061 (void)handle_futex_death(td, em, uaddr, pip, 1062 LINUX_HANDLE_DEATH_PENDING); 1063 } 1064 } 1065