1 // SPDX-License-Identifier: GPL-2.0-or-later
36  * sys_futex(WAIT, futex, val);
50  * This would cause the waiter on CPU 0 to wait forever because it
60  * sys_futex(WAIT, futex, val);
64  *   smp_mb(); (A) <-- paired with -.
73  *                                  `--------> smp_mb(); (B)
80  *     waiters--; (b)                        unlock(hash_bucket(futex));
99  * the wait call can return error, in which case we backtrack from it in (b).
104  * acquiring the lock. It then decrements them again  after releasing it -
112 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in __futex_wake_mark()
117 	 * The waiting task can free the futex_q as soon as q->lock_ptr = NULL  in __futex_wake_mark()
123 	smp_store_release(&q->lock_ptr, NULL);  in __futex_wake_mark()
136 	struct task_struct *p = q->task;  in futex_wake_mark()
147 	 * the hb->lock.  in futex_wake_mark()
164 		return -EINVAL;  in futex_wake()
179 	spin_lock(&hb->lock);  in futex_wake()
181 	plist_for_each_entry_safe(this, next, &hb->chain, list) {  in futex_wake()
182 		if (futex_match (&this->key, &key)) {  in futex_wake()
183 			if (this->pi_state || this->rt_waiter) {  in futex_wake()
184 				ret = -EINVAL;  in futex_wake()
189 			if (!(this->bitset & bitset))  in futex_wake()
192 			this->wake(&wake_q, this);  in futex_wake()
198 	spin_unlock(&hb->lock);  in futex_wake()
213 			char comm[sizeof(current->comm)];  in futex_atomic_op_inuser()
215 			 * kill this print and return -EINVAL when userspace  in futex_atomic_op_inuser()
245 		return -ENOSYS;  in futex_atomic_op_inuser()
262 retry:  in futex_wake_op()
280 		    unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {  in futex_wake_op()
289 		if (op_ret == -EFAULT) {  in futex_wake_op()
298 		goto retry;  in futex_wake_op()
301 	plist_for_each_entry_safe(this, next, &hb1->chain, list) {  in futex_wake_op()
302 		if (futex_match (&this->key, &key1)) {  in futex_wake_op()
303 			if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
304 				ret = -EINVAL;  in futex_wake_op()
307 			this->wake(&wake_q, this);  in futex_wake_op()
315 		plist_for_each_entry_safe(this, next, &hb2->chain, list) {  in futex_wake_op()
316 			if (futex_match (&this->key, &key2)) {  in futex_wake_op()
317 				if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
318 					ret = -EINVAL;  in futex_wake_op()
321 				this->wake(&wake_q, this);  in futex_wake_op()
338  * futex_wait_queue() - futex_queue() and wait for wakeup, timeout, or signal
361 	 * has tried to wake us, and we can skip the call to schedule().  in futex_wait_queue()
363 	if (likely(!plist_node_empty(&q->list))) {  in futex_wait_queue()
369 		if (!timeout || timeout->task)  in futex_wait_queue()
376  * futex_unqueue_multiple - Remove various futexes from their hash bucket
383  *  - >=0 - Index of the last futex that was awoken;
384  *  - -1  - No futex was awoken
388 	int ret = -1, i;  in futex_unqueue_multiple()
399  * futex_wait_multiple_setup - Prepare to wait and enqueue multiple futexes
400  * @vs:		The futex list to wait on
410  *  -  1 - One of the futexes was woken by another thread
411  *  -  0 - Success
412  *  - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL
417 	bool retry = false;  in futex_wait_multiple_setup()  local
425 	 * make sure that current->state is TASK_INTERRUPTIBLE, so we don't  in futex_wait_multiple_setup()
432 	 * Private futexes doesn't need to recalculate hash in retry, so skip  in futex_wait_multiple_setup()
435 retry:  in futex_wait_multiple_setup()
437 		if (!(vs[i].w.flags & FLAGS_SHARED) && retry)  in futex_wait_multiple_setup()
486 			 * retry all the work.  in futex_wait_multiple_setup()
489 				return -EFAULT;  in futex_wait_multiple_setup()
491 			retry = true;  in futex_wait_multiple_setup()
492 			goto retry;  in futex_wait_multiple_setup()
496 			return -EWOULDBLOCK;  in futex_wait_multiple_setup()
503  * futex_sleep_multiple - Check sleeping conditions and sleep
504  * @vs:    List of futexes to wait for
514 	if (to && !to->task)  in futex_sleep_multiple()
517 	for (; count; count--, vs++) {  in futex_sleep_multiple()
518 		if (!READ_ONCE(vs->q.lock_ptr))  in futex_sleep_multiple()
526  * futex_wait_multiple - Prepare to wait on and enqueue several futexes
527  * @vs:		The list of futexes to wait on
536  *  - >=0 - Hint to the futex that was awoken
537  *  - <0  - On error
565 		if (to && !to->task)  in futex_wait_multiple()
566 			return -ETIMEDOUT;  in futex_wait_multiple()
568 			return -ERESTARTSYS;  in futex_wait_multiple()
571 		 * which just retry.  in futex_wait_multiple()
577  * futex_wait_setup() - Prepare to wait on a futex
589  *  -  0 - uaddr contains val and hb has been locked;
590  *  - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
599 	 * Access the page AFTER the hash-bucket is locked.  in futex_wait_setup()
607 	 * any cond.  If we locked the hash-bucket after testing *uaddr, that  in futex_wait_setup()
611 	 * On the other hand, we insert q and release the hash-bucket only  in futex_wait_setup()
616 retry:  in futex_wait_setup()
617 	ret = get_futex_key(uaddr, flags, &q->key, FUTEX_READ);  in futex_wait_setup()
636 		goto retry;  in futex_wait_setup()
641 		ret = -EWOULDBLOCK;  in futex_wait_setup()
655 		return -EINVAL;  in __futex_wait()
659 retry:  in __futex_wait()
661 	 * Prepare to wait on uaddr. On success, it holds hb->lock and q  in __futex_wait()
668 	/* futex_queue and wait for wakeup, timeout, or a signal. */  in __futex_wait()
675 	if (to && !to->task)  in __futex_wait()
676 		return -ETIMEDOUT;  in __futex_wait()
683 		goto retry;  in __futex_wait()
685 	return -ERESTARTSYS;  in __futex_wait()
695 			       current->timer_slack_ns);  in futex_wait()
703 	hrtimer_cancel(&to->timer);  in futex_wait()
704 	destroy_hrtimer_on_stack(&to->timer);  in futex_wait()
706 	if (ret == -ERESTARTSYS) {  in futex_wait()
707 		restart = &current->restart_block;  in futex_wait()
708 		restart->futex.uaddr = uaddr;  in futex_wait()
709 		restart->futex.val = val;  in futex_wait()
710 		restart->futex.time = *abs_time;  in futex_wait()
711 		restart->futex.bitset = bitset;  in futex_wait()
712 		restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;  in futex_wait()
722 	u32 __user *uaddr = restart->futex.uaddr;  in futex_wait_restart()
725 	if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {  in futex_wait_restart()
726 		t = restart->futex.time;  in futex_wait_restart()
729 	restart->fn = do_no_restart_syscall;  in futex_wait_restart()
731 	return (long)futex_wait(uaddr, restart->futex.flags,  in futex_wait_restart()
732 				restart->futex.val, tp, restart->futex.bitset);  in futex_wait_restart()