1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2009-2021 Dmitry Chagin <dchagin@FreeBSD.org>
5 * Copyright (c) 2008 Roman Divacky
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/param.h>
30 #include <sys/imgact.h>
31 #include <sys/imgact_elf.h>
32 #include <sys/ktr.h>
33 #include <sys/lock.h>
34 #include <sys/mutex.h>
35 #include <sys/priv.h>
36 #include <sys/proc.h>
37 #include <sys/sched.h>
38 #include <sys/sysent.h>
39 #include <sys/vnode.h>
40 #include <sys/umtxvar.h>
41
42 #ifdef COMPAT_LINUX32
43 #include <machine/../linux32/linux.h>
44 #include <machine/../linux32/linux32_proto.h>
45 #else
46 #include <machine/../linux/linux.h>
47 #include <machine/../linux/linux_proto.h>
48 #endif
49 #include <compat/linux/linux_emul.h>
50 #include <compat/linux/linux_futex.h>
51 #include <compat/linux/linux_misc.h>
52 #include <compat/linux/linux_time.h>
53 #include <compat/linux/linux_util.h>
54
55 #define FUTEX_SHARED 0x8 /* shared futex */
56 #define FUTEX_UNOWNED 0
57
58 #define GET_SHARED(a) (a->flags & FUTEX_SHARED) ? AUTO_SHARE : THREAD_SHARE
59
60 static int futex_atomic_op(struct thread *, int, uint32_t *, int *);
61 static int handle_futex_death(struct thread *td, struct linux_emuldata *,
62 uint32_t *, unsigned int, bool);
63 static int fetch_robust_entry(struct linux_robust_list **,
64 struct linux_robust_list **, unsigned int *);
65
66 struct linux_futex_args {
67 uint32_t *uaddr;
68 int32_t op;
69 uint32_t flags;
70 bool clockrt;
71 uint32_t val;
72 struct timespec *ts;
73 uint32_t *uaddr2;
74 uint32_t val3;
75 bool val3_compare;
76 struct timespec kts;
77 };
78
79 static inline int futex_key_get(const void *, int, int, struct umtx_key *);
80 static void linux_umtx_abs_timeout_init(struct umtx_abs_timeout *,
81 struct linux_futex_args *);
82 static int linux_futex(struct thread *, struct linux_futex_args *);
83 static int linux_futex_wait(struct thread *, struct linux_futex_args *);
84 static int linux_futex_wake(struct thread *, struct linux_futex_args *);
85 static int linux_futex_requeue(struct thread *, struct linux_futex_args *);
86 static int linux_futex_wakeop(struct thread *, struct linux_futex_args *);
87 static int linux_futex_lock_pi(struct thread *, bool, struct linux_futex_args *);
88 static int linux_futex_unlock_pi(struct thread *, bool,
89 struct linux_futex_args *);
90 static int futex_wake_pi(struct thread *, uint32_t *, bool);
91
92 static int
futex_key_get(const void * uaddr,int type,int share,struct umtx_key * key)93 futex_key_get(const void *uaddr, int type, int share, struct umtx_key *key)
94 {
95
96 /* Check that futex address is a 32bit aligned. */
97 if (!__is_aligned(uaddr, sizeof(uint32_t)))
98 return (EINVAL);
99 return (umtx_key_get(uaddr, type, share, key));
100 }
101
102 int
futex_wake(struct thread * td,uint32_t * uaddr,int val,bool shared)103 futex_wake(struct thread *td, uint32_t *uaddr, int val, bool shared)
104 {
105 struct linux_futex_args args;
106
107 bzero(&args, sizeof(args));
108 args.op = LINUX_FUTEX_WAKE;
109 args.uaddr = uaddr;
110 args.flags = shared == true ? FUTEX_SHARED : 0;
111 args.val = val;
112 args.val3 = FUTEX_BITSET_MATCH_ANY;
113
114 return (linux_futex_wake(td, &args));
115 }
116
117 static int
futex_wake_pi(struct thread * td,uint32_t * uaddr,bool shared)118 futex_wake_pi(struct thread *td, uint32_t *uaddr, bool shared)
119 {
120 struct linux_futex_args args;
121
122 bzero(&args, sizeof(args));
123 args.op = LINUX_FUTEX_UNLOCK_PI;
124 args.uaddr = uaddr;
125 args.flags = shared == true ? FUTEX_SHARED : 0;
126
127 return (linux_futex_unlock_pi(td, true, &args));
128 }
129
130 static int
futex_atomic_op(struct thread * td,int encoded_op,uint32_t * uaddr,int * res)131 futex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr,
132 int *res)
133 {
134 int op = (encoded_op >> 28) & 7;
135 int cmp = (encoded_op >> 24) & 15;
136 int oparg = (encoded_op << 8) >> 20;
137 int cmparg = (encoded_op << 20) >> 20;
138 int oldval = 0, ret;
139
140 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
141 oparg = 1 << oparg;
142
143 switch (op) {
144 case FUTEX_OP_SET:
145 ret = futex_xchgl(oparg, uaddr, &oldval);
146 break;
147 case FUTEX_OP_ADD:
148 ret = futex_addl(oparg, uaddr, &oldval);
149 break;
150 case FUTEX_OP_OR:
151 ret = futex_orl(oparg, uaddr, &oldval);
152 break;
153 case FUTEX_OP_ANDN:
154 ret = futex_andl(~oparg, uaddr, &oldval);
155 break;
156 case FUTEX_OP_XOR:
157 ret = futex_xorl(oparg, uaddr, &oldval);
158 break;
159 default:
160 ret = ENOSYS;
161 break;
162 }
163
164 if (ret != 0)
165 return (ret);
166
167 switch (cmp) {
168 case FUTEX_OP_CMP_EQ:
169 *res = (oldval == cmparg);
170 break;
171 case FUTEX_OP_CMP_NE:
172 *res = (oldval != cmparg);
173 break;
174 case FUTEX_OP_CMP_LT:
175 *res = (oldval < cmparg);
176 break;
177 case FUTEX_OP_CMP_GE:
178 *res = (oldval >= cmparg);
179 break;
180 case FUTEX_OP_CMP_LE:
181 *res = (oldval <= cmparg);
182 break;
183 case FUTEX_OP_CMP_GT:
184 *res = (oldval > cmparg);
185 break;
186 default:
187 ret = ENOSYS;
188 }
189
190 return (ret);
191 }
192
193 static int
linux_futex(struct thread * td,struct linux_futex_args * args)194 linux_futex(struct thread *td, struct linux_futex_args *args)
195 {
196 struct linux_pemuldata *pem;
197 struct proc *p;
198
199 if (args->op & LINUX_FUTEX_PRIVATE_FLAG) {
200 args->flags = 0;
201 args->op &= ~LINUX_FUTEX_PRIVATE_FLAG;
202 } else
203 args->flags = FUTEX_SHARED;
204
205 args->clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME;
206 args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME;
207
208 if (args->clockrt &&
209 args->op != LINUX_FUTEX_WAIT_BITSET &&
210 args->op != LINUX_FUTEX_WAIT_REQUEUE_PI &&
211 args->op != LINUX_FUTEX_LOCK_PI2)
212 return (ENOSYS);
213
214 switch (args->op) {
215 case LINUX_FUTEX_WAIT:
216 args->val3 = FUTEX_BITSET_MATCH_ANY;
217 /* FALLTHROUGH */
218
219 case LINUX_FUTEX_WAIT_BITSET:
220 LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x",
221 args->uaddr, args->val, args->val3);
222
223 return (linux_futex_wait(td, args));
224
225 case LINUX_FUTEX_WAKE:
226 args->val3 = FUTEX_BITSET_MATCH_ANY;
227 /* FALLTHROUGH */
228
229 case LINUX_FUTEX_WAKE_BITSET:
230 LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x",
231 args->uaddr, args->val, args->val3);
232
233 return (linux_futex_wake(td, args));
234
235 case LINUX_FUTEX_REQUEUE:
236 /*
237 * Glibc does not use this operation since version 2.3.3,
238 * as it is racy and replaced by FUTEX_CMP_REQUEUE operation.
239 * Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when
240 * FUTEX_REQUEUE returned EINVAL.
241 */
242 pem = pem_find(td->td_proc);
243 if ((pem->flags & LINUX_XDEPR_REQUEUEOP) == 0) {
244 linux_msg(td, "unsupported FUTEX_REQUEUE");
245 pem->flags |= LINUX_XDEPR_REQUEUEOP;
246 }
247
248 /*
249 * The above is true, however musl libc does make use of the
250 * futex requeue operation, allow operation for brands which
251 * set LINUX_BI_FUTEX_REQUEUE bit of Brandinfo flags.
252 */
253 p = td->td_proc;
254 const Elf_Brandinfo *bi = p->p_elf_brandinfo;
255 if (bi == NULL || ((bi->flags & LINUX_BI_FUTEX_REQUEUE)) == 0)
256 return (EINVAL);
257 args->val3_compare = false;
258 /* FALLTHROUGH */
259
260 case LINUX_FUTEX_CMP_REQUEUE:
261 LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p "
262 "nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x",
263 args->uaddr, args->val, args->val3, args->uaddr2,
264 args->ts);
265
266 return (linux_futex_requeue(td, args));
267
268 case LINUX_FUTEX_WAKE_OP:
269 LINUX_CTR5(sys_futex, "WAKE_OP "
270 "uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x",
271 args->uaddr, args->val, args->uaddr2, args->val3,
272 args->ts);
273
274 return (linux_futex_wakeop(td, args));
275
276 case LINUX_FUTEX_LOCK_PI:
277 args->clockrt = true;
278 /* FALLTHROUGH */
279
280 case LINUX_FUTEX_LOCK_PI2:
281 LINUX_CTR2(sys_futex, "LOCKPI uaddr %p val 0x%x",
282 args->uaddr, args->val);
283
284 return (linux_futex_lock_pi(td, false, args));
285
286 case LINUX_FUTEX_UNLOCK_PI:
287 LINUX_CTR1(sys_futex, "UNLOCKPI uaddr %p",
288 args->uaddr);
289
290 return (linux_futex_unlock_pi(td, false, args));
291
292 case LINUX_FUTEX_TRYLOCK_PI:
293 LINUX_CTR1(sys_futex, "TRYLOCKPI uaddr %p",
294 args->uaddr);
295
296 return (linux_futex_lock_pi(td, true, args));
297
298 /*
299 * Current implementation of FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI
300 * can't be used anymore to implement conditional variables.
301 * A detailed explanation can be found here:
302 *
303 * https://sourceware.org/bugzilla/show_bug.cgi?id=13165
304 * and here http://austingroupbugs.net/view.php?id=609
305 *
306 * And since commit
307 * https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=ed19993b5b0d05d62cc883571519a67dae481a14
308 * glibc does not use them.
309 */
310 case LINUX_FUTEX_WAIT_REQUEUE_PI:
311 /* not yet implemented */
312 pem = pem_find(td->td_proc);
313 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
314 linux_msg(td, "unsupported FUTEX_WAIT_REQUEUE_PI");
315 pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
316 }
317 return (ENOSYS);
318
319 case LINUX_FUTEX_CMP_REQUEUE_PI:
320 /* not yet implemented */
321 pem = pem_find(td->td_proc);
322 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) {
323 linux_msg(td, "unsupported FUTEX_CMP_REQUEUE_PI");
324 pem->flags |= LINUX_XUNSUP_FUTEXPIOP;
325 }
326 return (ENOSYS);
327
328 default:
329 linux_msg(td, "unsupported futex op %d", args->op);
330 return (ENOSYS);
331 }
332 }
333
334 /*
335 * pi protocol:
336 * - 0 futex word value means unlocked.
337 * - TID futex word value means locked.
338 * Userspace uses atomic ops to lock/unlock these futexes without entering the
339 * kernel. If the lock-acquire fastpath fails, (transition from 0 to TID fails),
340 * then FUTEX_LOCK_PI is called.
341 * The kernel atomically set FUTEX_WAITERS bit in the futex word value, if no
342 * other waiters exists looks up the thread that owns the futex (it has put its
343 * own TID into the futex value) and made this thread the owner of the internal
344 * pi-aware lock object (mutex). Then the kernel tries to lock the internal lock
345 * object, on which it blocks. Once it returns, it has the mutex acquired, and it
346 * sets the futex value to its own TID and returns (futex value contains
347 * FUTEX_WAITERS|TID).
348 * The unlock fastpath would fail (because the FUTEX_WAITERS bit is set) and
349 * FUTEX_UNLOCK_PI will be called.
350 * If a futex is found to be held at exit time, the kernel sets the OWNER_DIED
351 * bit of the futex word and wakes up the next futex waiter (if any), WAITERS
352 * bit is preserved (if any).
353 * If OWNER_DIED bit is set the kernel sanity checks the futex word value against
354 * the internal futex state and if correct, acquire futex.
355 */
356 static int
linux_futex_lock_pi(struct thread * td,bool try,struct linux_futex_args * args)357 linux_futex_lock_pi(struct thread *td, bool try, struct linux_futex_args *args)
358 {
359 struct umtx_abs_timeout timo, *timop;
360 struct linux_emuldata *em;
361 struct umtx_pi *pi, *new_pi;
362 struct thread *td1;
363 struct umtx_q *uq;
364 int error, rv;
365 uint32_t owner, old_owner;
366
367 em = em_find(td);
368 uq = td->td_umtxq;
369 error = futex_key_get(args->uaddr, TYPE_PI_FUTEX, GET_SHARED(args),
370 &uq->uq_key);
371 if (error != 0)
372 return (error);
373 if (!try && args->ts != NULL) {
374 timop = &timo;
375 linux_umtx_abs_timeout_init(timop, args);
376 } else {
377 timop = NULL;
378 }
379
380 umtxq_lock(&uq->uq_key);
381 pi = umtx_pi_lookup(&uq->uq_key);
382 if (pi == NULL) {
383 new_pi = umtx_pi_alloc(M_NOWAIT);
384 if (new_pi == NULL) {
385 umtxq_unlock(&uq->uq_key);
386 new_pi = umtx_pi_alloc(M_WAITOK);
387 umtxq_lock(&uq->uq_key);
388 pi = umtx_pi_lookup(&uq->uq_key);
389 if (pi != NULL) {
390 umtx_pi_free(new_pi);
391 new_pi = NULL;
392 }
393 }
394 if (new_pi != NULL) {
395 new_pi->pi_key = uq->uq_key;
396 umtx_pi_insert(new_pi);
397 pi = new_pi;
398 }
399 }
400 umtx_pi_ref(pi);
401 umtxq_unlock(&uq->uq_key);
402 for (;;) {
403 /* Try uncontested case first. */
404 rv = casueword32(args->uaddr, FUTEX_UNOWNED, &owner, em->em_tid);
405 /* The acquire succeeded. */
406 if (rv == 0) {
407 error = 0;
408 break;
409 }
410 if (rv == -1) {
411 error = EFAULT;
412 break;
413 }
414
415 /*
416 * Nobody owns it, but the acquire failed. This can happen
417 * with ll/sc atomic.
418 */
419 if (owner == FUTEX_UNOWNED) {
420 error = thread_check_susp(td, true);
421 if (error != 0)
422 break;
423 continue;
424 }
425
426 /*
427 * Avoid overwriting a possible error from sleep due
428 * to the pending signal with suspension check result.
429 */
430 if (error == 0) {
431 error = thread_check_susp(td, true);
432 if (error != 0)
433 break;
434 }
435
436 /* The futex word at *uaddr is already locked by the caller. */
437 if ((owner & FUTEX_TID_MASK) == em->em_tid) {
438 error = EDEADLK;
439 break;
440 }
441
442 /*
443 * Futex owner died, handle_futex_death() set the OWNER_DIED bit
444 * and clear tid. Try to acquire it.
445 */
446 if ((owner & FUTEX_TID_MASK) == FUTEX_UNOWNED) {
447 old_owner = owner;
448 owner = owner & (FUTEX_WAITERS | FUTEX_OWNER_DIED);
449 owner |= em->em_tid;
450 rv = casueword32(args->uaddr, old_owner, &owner, owner);
451 if (rv == -1) {
452 error = EFAULT;
453 break;
454 }
455 if (rv == 1) {
456 if (error == 0) {
457 error = thread_check_susp(td, true);
458 if (error != 0)
459 break;
460 }
461
462 /*
463 * If this failed the lock could
464 * changed, restart.
465 */
466 continue;
467 }
468
469 umtxq_lock(&uq->uq_key);
470 umtxq_busy(&uq->uq_key);
471 error = umtx_pi_claim(pi, td);
472 umtxq_unbusy(&uq->uq_key);
473 umtxq_unlock(&uq->uq_key);
474 if (error != 0) {
475 /*
476 * Since we're going to return an
477 * error, restore the futex to its
478 * previous, unowned state to avoid
479 * compounding the problem.
480 */
481 (void)casuword32(args->uaddr, owner, old_owner);
482 }
483 break;
484 }
485
486 /*
487 * Inconsistent state: OWNER_DIED is set and tid is not 0.
488 * Linux does some checks of futex state, we return EINVAL,
489 * as the user space can take care of this.
490 */
491 if ((owner & FUTEX_OWNER_DIED) != FUTEX_UNOWNED) {
492 error = EINVAL;
493 break;
494 }
495
496 if (try != 0) {
497 error = EBUSY;
498 break;
499 }
500
501 /*
502 * If we caught a signal, we have retried and now
503 * exit immediately.
504 */
505 if (error != 0)
506 break;
507
508 umtxq_busy_unlocked(&uq->uq_key);
509
510 /*
511 * Set the contested bit so that a release in user space knows
512 * to use the system call for unlock. If this fails either some
513 * one else has acquired the lock or it has been released.
514 */
515 rv = casueword32(args->uaddr, owner, &owner,
516 owner | FUTEX_WAITERS);
517 if (rv == -1) {
518 umtxq_unbusy_unlocked(&uq->uq_key);
519 error = EFAULT;
520 break;
521 }
522 if (rv == 1) {
523 umtxq_unbusy_unlocked(&uq->uq_key);
524 error = thread_check_susp(td, true);
525 if (error != 0)
526 break;
527
528 /*
529 * The lock changed and we need to retry or we
530 * lost a race to the thread unlocking the umtx.
531 */
532 continue;
533 }
534
535 /*
536 * Substitute Linux thread id by native thread id to
537 * avoid refactoring code of umtxq_sleep_pi().
538 */
539 td1 = linux_tdfind(td, owner & FUTEX_TID_MASK, -1);
540 if (td1 != NULL) {
541 owner = td1->td_tid;
542 PROC_UNLOCK(td1->td_proc);
543 } else {
544 umtxq_unbusy_unlocked(&uq->uq_key);
545 error = EINVAL;
546 break;
547 }
548
549 umtxq_lock(&uq->uq_key);
550
551 /* We set the contested bit, sleep. */
552 error = umtxq_sleep_pi(uq, pi, owner, "futexp",
553 timop, (args->flags & FUTEX_SHARED) != 0);
554 if (error != 0)
555 continue;
556
557 error = thread_check_susp(td, false);
558 if (error != 0)
559 break;
560 }
561
562 umtxq_lock(&uq->uq_key);
563 umtx_pi_unref(pi);
564 umtxq_unlock(&uq->uq_key);
565 umtx_key_release(&uq->uq_key);
566 return (error);
567 }
568
569 static int
linux_futex_unlock_pi(struct thread * td,bool rb,struct linux_futex_args * args)570 linux_futex_unlock_pi(struct thread *td, bool rb, struct linux_futex_args *args)
571 {
572 struct linux_emuldata *em;
573 struct umtx_key key;
574 uint32_t old, owner, new_owner;
575 int count, error;
576
577 em = em_find(td);
578
579 /*
580 * Make sure we own this mtx.
581 */
582 error = fueword32(args->uaddr, &owner);
583 if (error == -1)
584 return (EFAULT);
585 if (!rb && (owner & FUTEX_TID_MASK) != em->em_tid)
586 return (EPERM);
587
588 error = futex_key_get(args->uaddr, TYPE_PI_FUTEX, GET_SHARED(args), &key);
589 if (error != 0)
590 return (error);
591 umtxq_lock(&key);
592 umtxq_busy(&key);
593 error = umtx_pi_drop(td, &key, rb, &count);
594 if (error != 0 || rb) {
595 umtxq_unbusy(&key);
596 umtxq_unlock(&key);
597 umtx_key_release(&key);
598 return (error);
599 }
600 umtxq_unlock(&key);
601
602 /*
603 * When unlocking the futex, it must be marked as unowned if
604 * there is zero or one thread only waiting for it.
605 * Otherwise, it must be marked as contested.
606 */
607 if (count > 1)
608 new_owner = FUTEX_WAITERS;
609 else
610 new_owner = FUTEX_UNOWNED;
611
612 again:
613 error = casueword32(args->uaddr, owner, &old, new_owner);
614 if (error == 1) {
615 error = thread_check_susp(td, false);
616 if (error == 0)
617 goto again;
618 }
619 umtxq_unbusy_unlocked(&key);
620 umtx_key_release(&key);
621 if (error == -1)
622 return (EFAULT);
623 if (error == 0 && old != owner)
624 return (EINVAL);
625 return (error);
626 }
627
628 static int
linux_futex_wakeop(struct thread * td,struct linux_futex_args * args)629 linux_futex_wakeop(struct thread *td, struct linux_futex_args *args)
630 {
631 struct umtx_key key, key2;
632 int nrwake, op_ret, ret;
633 int error, count;
634
635 if (args->uaddr == args->uaddr2)
636 return (EINVAL);
637
638 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args), &key);
639 if (error != 0)
640 return (error);
641 error = futex_key_get(args->uaddr2, TYPE_FUTEX, GET_SHARED(args), &key2);
642 if (error != 0) {
643 umtx_key_release(&key);
644 return (error);
645 }
646 umtxq_busy_unlocked(&key);
647 error = futex_atomic_op(td, args->val3, args->uaddr2, &op_ret);
648 umtxq_lock(&key);
649 umtxq_unbusy(&key);
650 if (error != 0)
651 goto out;
652 ret = umtxq_signal_mask(&key, args->val, args->val3);
653 if (op_ret > 0) {
654 nrwake = (int)(unsigned long)args->ts;
655 umtxq_lock(&key2);
656 count = umtxq_count(&key2);
657 if (count > 0)
658 ret += umtxq_signal_mask(&key2, nrwake, args->val3);
659 else
660 ret += umtxq_signal_mask(&key, nrwake, args->val3);
661 umtxq_unlock(&key2);
662 }
663 td->td_retval[0] = ret;
664 out:
665 umtxq_unlock(&key);
666 umtx_key_release(&key2);
667 umtx_key_release(&key);
668 return (error);
669 }
670
671 static int
linux_futex_requeue(struct thread * td,struct linux_futex_args * args)672 linux_futex_requeue(struct thread *td, struct linux_futex_args *args)
673 {
674 int nrwake, nrrequeue;
675 struct umtx_key key, key2;
676 int error;
677 uint32_t uval;
678
679 /*
680 * Linux allows this, we would not, it is an incorrect
681 * usage of declared ABI, so return EINVAL.
682 */
683 if (args->uaddr == args->uaddr2)
684 return (EINVAL);
685
686 nrrequeue = (int)(unsigned long)args->ts;
687 nrwake = args->val;
688 /*
689 * Sanity check to prevent signed integer overflow,
690 * see Linux CVE-2018-6927
691 */
692 if (nrwake < 0 || nrrequeue < 0)
693 return (EINVAL);
694
695 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args), &key);
696 if (error != 0)
697 return (error);
698 error = futex_key_get(args->uaddr2, TYPE_FUTEX, GET_SHARED(args), &key2);
699 if (error != 0) {
700 umtx_key_release(&key);
701 return (error);
702 }
703 umtxq_busy_unlocked(&key);
704 error = fueword32(args->uaddr, &uval);
705 if (error != 0)
706 error = EFAULT;
707 else if (args->val3_compare == true && uval != args->val3)
708 error = EWOULDBLOCK;
709 umtxq_lock(&key);
710 umtxq_unbusy(&key);
711 if (error == 0) {
712 umtxq_lock(&key2);
713 td->td_retval[0] = umtxq_requeue(&key, nrwake, &key2, nrrequeue);
714 umtxq_unlock(&key2);
715 }
716 umtxq_unlock(&key);
717 umtx_key_release(&key2);
718 umtx_key_release(&key);
719 return (error);
720 }
721
722 static int
linux_futex_wake(struct thread * td,struct linux_futex_args * args)723 linux_futex_wake(struct thread *td, struct linux_futex_args *args)
724 {
725 struct umtx_key key;
726 int error;
727
728 if (args->val3 == 0)
729 return (EINVAL);
730
731 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args), &key);
732 if (error != 0)
733 return (error);
734 umtxq_lock(&key);
735 td->td_retval[0] = umtxq_signal_mask(&key, args->val, args->val3);
736 umtxq_unlock(&key);
737 umtx_key_release(&key);
738 return (0);
739 }
740
741 static int
linux_futex_wait(struct thread * td,struct linux_futex_args * args)742 linux_futex_wait(struct thread *td, struct linux_futex_args *args)
743 {
744 struct umtx_abs_timeout timo;
745 struct umtx_q *uq;
746 uint32_t uval;
747 int error;
748
749 if (args->val3 == 0)
750 error = EINVAL;
751
752 uq = td->td_umtxq;
753 error = futex_key_get(args->uaddr, TYPE_FUTEX, GET_SHARED(args),
754 &uq->uq_key);
755 if (error != 0)
756 return (error);
757 if (args->ts != NULL)
758 linux_umtx_abs_timeout_init(&timo, args);
759 umtxq_lock(&uq->uq_key);
760 umtxq_busy(&uq->uq_key);
761 uq->uq_bitset = args->val3;
762 umtxq_insert(uq);
763 umtxq_unlock(&uq->uq_key);
764 error = fueword32(args->uaddr, &uval);
765 if (error != 0)
766 error = EFAULT;
767 else if (uval != args->val)
768 error = EWOULDBLOCK;
769 umtxq_lock(&uq->uq_key);
770 umtxq_unbusy(&uq->uq_key);
771 if (error == 0) {
772 error = umtxq_sleep(uq, "futex",
773 args->ts == NULL ? NULL : &timo);
774 if ((uq->uq_flags & UQF_UMTXQ) == 0)
775 error = 0;
776 else
777 umtxq_remove(uq);
778 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
779 umtxq_remove(uq);
780 }
781 umtxq_unlock(&uq->uq_key);
782 umtx_key_release(&uq->uq_key);
783 if (error == ERESTART)
784 error = EINTR;
785 return (error);
786 }
787
788 static void
linux_umtx_abs_timeout_init(struct umtx_abs_timeout * timo,struct linux_futex_args * args)789 linux_umtx_abs_timeout_init(struct umtx_abs_timeout *timo,
790 struct linux_futex_args *args)
791 {
792 int clockid, absolute;
793
794 /*
795 * The FUTEX_CLOCK_REALTIME option bit can be employed only with the
796 * FUTEX_WAIT_BITSET, FUTEX_WAIT_REQUEUE_PI, FUTEX_LOCK_PI2.
797 * For FUTEX_WAIT, timeout is interpreted as a relative value, for other
798 * futex operations timeout is interpreted as an absolute value.
799 * If FUTEX_CLOCK_REALTIME option bit is set, the Linux kernel measures
800 * the timeout against the CLOCK_REALTIME clock, otherwise the kernel
801 * measures the timeout against the CLOCK_MONOTONIC clock.
802 */
803 clockid = args->clockrt ? CLOCK_REALTIME : CLOCK_MONOTONIC;
804 absolute = args->op == LINUX_FUTEX_WAIT ? false : true;
805 umtx_abs_timeout_init(timo, clockid, absolute, args->ts);
806 }
807
808 int
linux_sys_futex(struct thread * td,struct linux_sys_futex_args * args)809 linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args)
810 {
811 struct linux_futex_args fargs = {
812 .uaddr = args->uaddr,
813 .op = args->op,
814 .val = args->val,
815 .ts = NULL,
816 .uaddr2 = args->uaddr2,
817 .val3 = args->val3,
818 .val3_compare = true,
819 };
820 int error;
821
822 switch (args->op & LINUX_FUTEX_CMD_MASK) {
823 case LINUX_FUTEX_WAIT:
824 case LINUX_FUTEX_WAIT_BITSET:
825 case LINUX_FUTEX_LOCK_PI:
826 case LINUX_FUTEX_LOCK_PI2:
827 if (args->timeout != NULL) {
828 error = linux_get_timespec(&fargs.kts, args->timeout);
829 if (error != 0)
830 return (error);
831 fargs.ts = &fargs.kts;
832 }
833 break;
834 default:
835 fargs.ts = PTRIN(args->timeout);
836 }
837 return (linux_futex(td, &fargs));
838 }
839
840 #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32))
841 int
linux_sys_futex_time64(struct thread * td,struct linux_sys_futex_time64_args * args)842 linux_sys_futex_time64(struct thread *td,
843 struct linux_sys_futex_time64_args *args)
844 {
845 struct linux_futex_args fargs = {
846 .uaddr = args->uaddr,
847 .op = args->op,
848 .val = args->val,
849 .ts = NULL,
850 .uaddr2 = args->uaddr2,
851 .val3 = args->val3,
852 .val3_compare = true,
853 };
854 int error;
855
856 switch (args->op & LINUX_FUTEX_CMD_MASK) {
857 case LINUX_FUTEX_WAIT:
858 case LINUX_FUTEX_WAIT_BITSET:
859 case LINUX_FUTEX_LOCK_PI:
860 case LINUX_FUTEX_LOCK_PI2:
861 if (args->timeout != NULL) {
862 error = linux_get_timespec64(&fargs.kts, args->timeout);
863 if (error != 0)
864 return (error);
865 fargs.ts = &fargs.kts;
866 }
867 break;
868 default:
869 fargs.ts = PTRIN(args->timeout);
870 }
871 return (linux_futex(td, &fargs));
872 }
873 #endif
874
875 int
linux_set_robust_list(struct thread * td,struct linux_set_robust_list_args * args)876 linux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args)
877 {
878 struct linux_emuldata *em;
879
880 if (args->len != sizeof(struct linux_robust_list_head))
881 return (EINVAL);
882
883 em = em_find(td);
884 em->robust_futexes = args->head;
885
886 return (0);
887 }
888
889 int
linux_get_robust_list(struct thread * td,struct linux_get_robust_list_args * args)890 linux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args)
891 {
892 struct linux_emuldata *em;
893 struct linux_robust_list_head *head;
894 l_size_t len;
895 struct thread *td2;
896 int error;
897
898 if (!args->pid) {
899 em = em_find(td);
900 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
901 head = em->robust_futexes;
902 } else {
903 td2 = linux_tdfind(td, args->pid, -1);
904 if (td2 == NULL)
905 return (ESRCH);
906 if (SV_PROC_ABI(td2->td_proc) != SV_ABI_LINUX) {
907 PROC_UNLOCK(td2->td_proc);
908 return (EPERM);
909 }
910
911 em = em_find(td2);
912 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n"));
913 /* XXX: ptrace? */
914 if (priv_check(td, PRIV_CRED_SETUID) ||
915 priv_check(td, PRIV_CRED_SETEUID) ||
916 p_candebug(td, td2->td_proc)) {
917 PROC_UNLOCK(td2->td_proc);
918 return (EPERM);
919 }
920 head = em->robust_futexes;
921
922 PROC_UNLOCK(td2->td_proc);
923 }
924
925 len = sizeof(struct linux_robust_list_head);
926 error = copyout(&len, args->len, sizeof(l_size_t));
927 if (error != 0)
928 return (EFAULT);
929
930 return (copyout(&head, args->head, sizeof(l_uintptr_t)));
931 }
932
933 static int
handle_futex_death(struct thread * td,struct linux_emuldata * em,uint32_t * uaddr,unsigned int pi,bool pending_op)934 handle_futex_death(struct thread *td, struct linux_emuldata *em, uint32_t *uaddr,
935 unsigned int pi, bool pending_op)
936 {
937 uint32_t uval, nval, mval;
938 int error;
939
940 retry:
941 error = fueword32(uaddr, &uval);
942 if (error != 0)
943 return (EFAULT);
944
945 /*
946 * Special case for regular (non PI) futexes. The unlock path in
947 * user space has two race scenarios:
948 *
949 * 1. The unlock path releases the user space futex value and
950 * before it can execute the futex() syscall to wake up
951 * waiters it is killed.
952 *
953 * 2. A woken up waiter is killed before it can acquire the
954 * futex in user space.
955 *
956 * In both cases the TID validation below prevents a wakeup of
957 * potential waiters which can cause these waiters to block
958 * forever.
959 *
960 * In both cases it is safe to attempt waking up a potential
961 * waiter without touching the user space futex value and trying
962 * to set the OWNER_DIED bit.
963 */
964 if (pending_op && !pi && !uval) {
965 (void)futex_wake(td, uaddr, 1, true);
966 return (0);
967 }
968
969 if ((uval & FUTEX_TID_MASK) == em->em_tid) {
970 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
971 error = casueword32(uaddr, uval, &nval, mval);
972 if (error == -1)
973 return (EFAULT);
974 if (error == 1) {
975 error = thread_check_susp(td, false);
976 if (error != 0)
977 return (error);
978 goto retry;
979 }
980
981 if (!pi && (uval & FUTEX_WAITERS)) {
982 error = futex_wake(td, uaddr, 1, true);
983 if (error != 0)
984 return (error);
985 } else if (pi && (uval & FUTEX_WAITERS)) {
986 error = futex_wake_pi(td, uaddr, true);
987 if (error != 0)
988 return (error);
989 }
990 }
991
992 return (0);
993 }
994
995 static int
fetch_robust_entry(struct linux_robust_list ** entry,struct linux_robust_list ** head,unsigned int * pi)996 fetch_robust_entry(struct linux_robust_list **entry,
997 struct linux_robust_list **head, unsigned int *pi)
998 {
999 l_ulong uentry;
1000 int error;
1001
1002 error = copyin((const void *)head, &uentry, sizeof(uentry));
1003 if (error != 0)
1004 return (EFAULT);
1005
1006 *entry = (void *)(uentry & ~1UL);
1007 *pi = uentry & 1;
1008
1009 return (0);
1010 }
1011
1012 #define LINUX_HANDLE_DEATH_PENDING true
1013 #define LINUX_HANDLE_DEATH_LIST false
1014
1015 /* This walks the list of robust futexes releasing them. */
1016 void
release_futexes(struct thread * td,struct linux_emuldata * em)1017 release_futexes(struct thread *td, struct linux_emuldata *em)
1018 {
1019 struct linux_robust_list_head *head;
1020 struct linux_robust_list *entry, *next_entry, *pending;
1021 unsigned int limit = 2048, pi, next_pi, pip;
1022 uint32_t *uaddr;
1023 l_long futex_offset;
1024 int error;
1025
1026 head = em->robust_futexes;
1027 if (head == NULL)
1028 return;
1029
1030 if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi))
1031 return;
1032
1033 error = copyin(&head->futex_offset, &futex_offset,
1034 sizeof(futex_offset));
1035 if (error != 0)
1036 return;
1037
1038 if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip))
1039 return;
1040
1041 while (entry != &head->list) {
1042 error = fetch_robust_entry(&next_entry, PTRIN(&entry->next),
1043 &next_pi);
1044
1045 /*
1046 * A pending lock might already be on the list, so
1047 * don't process it twice.
1048 */
1049 if (entry != pending) {
1050 uaddr = (uint32_t *)((caddr_t)entry + futex_offset);
1051 if (handle_futex_death(td, em, uaddr, pi,
1052 LINUX_HANDLE_DEATH_LIST))
1053 return;
1054 }
1055 if (error != 0)
1056 return;
1057
1058 entry = next_entry;
1059 pi = next_pi;
1060
1061 if (!--limit)
1062 break;
1063
1064 sched_relinquish(curthread);
1065 }
1066
1067 if (pending) {
1068 uaddr = (uint32_t *)((caddr_t)pending + futex_offset);
1069 (void)handle_futex_death(td, em, uaddr, pip,
1070 LINUX_HANDLE_DEATH_PENDING);
1071 }
1072 }
1073