1 // SPDX-License-Identifier: GPL-2.0-or-later
12  * just woken up (timeout, signal). After the wake up the task has to
26  *   Q_REQUEUE_PI_NONE		-> Q_REQUEUE_PI_IGNORE
27  *   Q_REQUEUE_PI_IN_PROGRESS	-> Q_REQUEUE_PI_WAIT
30  *   Q_REQUEUE_PI_NONE		-> Q_REQUEUE_PI_INPROGRESS
31  *   Q_REQUEUE_PI_IN_PROGRESS	-> Q_REQUEUE_PI_DONE/LOCKED
32  *   Q_REQUEUE_PI_IN_PROGRESS	-> Q_REQUEUE_PI_NONE (requeue failed)
33  *   Q_REQUEUE_PI_WAIT		-> Q_REQUEUE_PI_DONE/LOCKED
34  *   Q_REQUEUE_PI_WAIT		-> Q_REQUEUE_PI_IGNORE (requeue failed)
62 	.wake		= futex_wake_mark,
69  * requeue_futex() - Requeue a futex_q from one hb to another
84 	if (likely(&hb1->chain != &hb2->chain)) {  in requeue_futex()
85 		plist_del(&q->list, &hb1->chain);  in requeue_futex()
88 		plist_add(&q->list, &hb2->chain);  in requeue_futex()
89 		q->lock_ptr = &hb2->lock;  in requeue_futex()
91 	q->key = *key2;  in requeue_futex()
104 	old = atomic_read_acquire(&q->requeue_state);  in futex_requeue_pi_prepare()
111 		 * IN_PROGRESS and a interleaved early wake to WAIT.  in futex_requeue_pi_prepare()
121 	} while (!atomic_try_cmpxchg(&q->requeue_state, &old, new));  in futex_requeue_pi_prepare()
123 	q->pi_state = pi_state;  in futex_requeue_pi_prepare()
131 	old = atomic_read_acquire(&q->requeue_state);  in futex_requeue_pi_complete()
149 	} while (!atomic_try_cmpxchg(&q->requeue_state, &old, new));  in futex_requeue_pi_complete()
154 		rcuwait_wake_up(&q->requeue_wait);  in futex_requeue_pi_complete()
162 	old = atomic_read_acquire(&q->requeue_state);  in futex_requeue_pi_wakeup_sync()
170 		 * the waiter or to wake it up once the requeue is done.  in futex_requeue_pi_wakeup_sync()
175 	} while (!atomic_try_cmpxchg(&q->requeue_state, &old, new));  in futex_requeue_pi_wakeup_sync()
180 		rcuwait_wait_event(&q->requeue_wait,  in futex_requeue_pi_wakeup_sync()
181 				   atomic_read(&q->requeue_state) != Q_REQUEUE_PI_WAIT,  in futex_requeue_pi_wakeup_sync()
184 		(void)atomic_cond_read_relaxed(&q->requeue_state, VAL != Q_REQUEUE_PI_WAIT);  in futex_requeue_pi_wakeup_sync()
191 	 * will modify q->requeue_state after this point.  in futex_requeue_pi_wakeup_sync()
193 	return atomic_read(&q->requeue_state);  in futex_requeue_pi_wakeup_sync()
197  * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue
210  * 3) Set @q::rt_waiter to NULL so the woken up task can detect atomic lock
213  * 4) Set the q->lock_ptr to the requeue target hb->lock for the case that
218  *    case that the pi state does not have to be fixed up.
220  * 6) Wake the waiter task.
222  * Must be called with both q->lock_ptr and hb->lock held.
228 	q->key = *key;  in requeue_pi_wake_futex()
232 	WARN_ON(!q->rt_waiter);  in requeue_pi_wake_futex()
233 	q->rt_waiter = NULL;  in requeue_pi_wake_futex()
235 	q->lock_ptr = &hb->lock;  in requeue_pi_wake_futex()
239 	wake_up_state(q->task, TASK_NORMAL);  in requeue_pi_wake_futex()
243  * futex_proxy_trylock_atomic() - Attempt an atomic lock for the top waiter
255  * Wake the top waiter if we succeed.  If the caller specified set_waiters,
259  * @exiting is only set when the return value is -EBUSY. If so, this holds
264  *  -  0 - failed to acquire the lock atomically;
265  *  - >0 - acquired the lock, return value is vpid of the top_waiter
266  *  - <0 - error
279 		return -EFAULT;  in futex_proxy_trylock_atomic()
282 		return -EFAULT;  in futex_proxy_trylock_atomic()
302 	if (!top_waiter->rt_waiter || top_waiter->pi_state)  in futex_proxy_trylock_atomic()
303 		return -EINVAL;  in futex_proxy_trylock_atomic()
306 	if (!futex_match(top_waiter->requeue_pi_key, key2))  in futex_proxy_trylock_atomic()
307 		return -EINVAL;  in futex_proxy_trylock_atomic()
311 		return -EAGAIN;  in futex_proxy_trylock_atomic()
319 	 * the new owner (@top_waiter->task) when @set_waiters is true.  in futex_proxy_trylock_atomic()
321 	ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,  in futex_proxy_trylock_atomic()
326 		 * attached to @top_waiter->task. That means state is fully  in futex_proxy_trylock_atomic()
338 		 * state. top_waiter::requeue_state cannot be fixed up here  in futex_proxy_trylock_atomic()
349  * futex_requeue() - Requeue waiters from uaddr1 to uaddr2
354  * @nr_wake:	number of waiters to wake (must be 1 for requeue_pi)
355  * @nr_requeue:	number of waiters to requeue (0-INT_MAX)
357  * @requeue_pi:	if we are attempting to requeue from a non-pi futex to a
364  *  - >=0 - on success, the number of tasks requeued or woken;
365  *  -  <0 - on error
379 		return -EINVAL;  in futex_requeue()
382 	 * When PI not supported: return -ENOSYS if requeue_pi is true,  in futex_requeue()
388 		return -ENOSYS;  in futex_requeue()
396 			return -EINVAL;  in futex_requeue()
400 		 * of waiters to wake up via the @nr_wake argument. With  in futex_requeue()
401 		 * REQUEUE_PI, waking up more than one waiter is creating  in futex_requeue()
402 		 * more problems than it solves. Waking up a waiter makes  in futex_requeue()
409 		 * look up pi_state and do more or less all the handling  in futex_requeue()
411 		 * waiters. So restrict the number of waiters to wake to  in futex_requeue()
412 		 * one, and only wake it up when the PI futex is  in futex_requeue()
420 			return -EINVAL;  in futex_requeue()
427 			return -ENOMEM;  in futex_requeue()
444 		return -EINVAL;  in futex_requeue()
472 			ret = -EAGAIN;  in futex_requeue()
481 		 * Attempt to acquire uaddr2 and wake the top waiter. If we  in futex_requeue()
498 		 * The top waiter's requeue_state is up to date:  in futex_requeue()
500 		 *  - If the lock was acquired atomically (ret == 1), then  in futex_requeue()
503 		 *    The top waiter has been dequeued and woken up and can  in futex_requeue()
513 		 *  - If the trylock failed with an error (ret < 0) then  in futex_requeue()
518 		 *  - If the trylock did not succeed (ret == 0) then the  in futex_requeue()
521 		 *    This will be cleaned up in the loop below, which  in futex_requeue()
544 		case -EFAULT:  in futex_requeue()
551 		case -EBUSY:  in futex_requeue()
552 		case -EAGAIN:  in futex_requeue()
555 			 * - EBUSY: Owner is exiting and we just wait for the  in futex_requeue()
557 			 * - EAGAIN: The user space value changed.  in futex_requeue()
574 	plist_for_each_entry_safe(this, next, &hb1->chain, list) {  in futex_requeue()
575 		if (task_count - nr_wake >= nr_requeue)  in futex_requeue()
578 		if (!futex_match(&this->key, &key1))  in futex_requeue()
588 		if ((requeue_pi && !this->rt_waiter) ||  in futex_requeue()
589 		    (!requeue_pi && this->rt_waiter) ||  in futex_requeue()
590 		    this->pi_state) {  in futex_requeue()
591 			ret = -EINVAL;  in futex_requeue()
595 		/* Plain futexes just wake or requeue and are done */  in futex_requeue()
598 				this->wake(&wake_q, this);  in futex_requeue()
605 		if (!futex_match(this->requeue_pi_key, &key2)) {  in futex_requeue()
606 			ret = -EINVAL;  in futex_requeue()
625 			 * next waiter. @this->pi_state is still NULL.  in futex_requeue()
631 		ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,  in futex_requeue()
632 						this->rt_waiter,  in futex_requeue()
633 						this->task);  in futex_requeue()
638 			 * on pi_state nor clear this->pi_state because the  in futex_requeue()
639 			 * waiter needs the pi_state for cleaning up the  in futex_requeue()
659 			this->pi_state = NULL;  in futex_requeue()
684  * handle_early_requeue_pi_wakeup() - Handle early wakeup on the initial futex
692  *  -EWOULDBLOCK or -ETIMEDOUT or -ERESTARTNOINTR
708 	WARN_ON_ONCE(&hb->lock != q->lock_ptr);  in handle_early_requeue_pi_wakeup()
714 	plist_del(&q->list, &hb->chain);  in handle_early_requeue_pi_wakeup()
718 	ret = -EWOULDBLOCK;  in handle_early_requeue_pi_wakeup()
719 	if (timeout && !timeout->task)  in handle_early_requeue_pi_wakeup()
720 		ret = -ETIMEDOUT;  in handle_early_requeue_pi_wakeup()
722 		ret = -ERESTARTNOINTR;  in handle_early_requeue_pi_wakeup()
727  * futex_wait_requeue_pi() - Wait on uaddr and take uaddr2
728  * @uaddr:	the futex we initially wait on (non-pi)
734  * @uaddr2:	the pi futex we will take prior to returning to user-space
737  * uaddr2 which must be PI aware and unique from uaddr.  Normal wakeup will wake
744  * via the following--
750  * If 3, cleanup and return -ERESTARTNOINTR.
758  * If 6, return -EWOULDBLOCK (restarting the syscall would do the same).
760  * If 4 or 7, we cleanup and return with -ETIMEDOUT.
763  *  -  0 - On success;
764  *  - <0 - On error
779 		return -ENOSYS;  in futex_wait_requeue_pi()
782 		return -EINVAL;  in futex_wait_requeue_pi()
785 		return -EINVAL;  in futex_wait_requeue_pi()
788 			       current->timer_slack_ns);  in futex_wait_requeue_pi()
805 	 * Prepare to wait on uaddr. On success, it holds hb->lock and q  in futex_wait_requeue_pi()
818 		ret = -EINVAL;  in futex_wait_requeue_pi()
828 		spin_lock(&hb->lock);  in futex_wait_requeue_pi()
830 		spin_unlock(&hb->lock);  in futex_wait_requeue_pi()
835 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
845 			 * Adjust the return value. It's either -EFAULT or  in futex_wait_requeue_pi()
854 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()
872 		 * acquired the lock, clear -ETIMEDOUT or -EINTR.  in futex_wait_requeue_pi()
880 		if (ret == -EINTR) {  in futex_wait_requeue_pi()
886 			 * -EWOULDBLOCK.  Save the overhead of the restart  in futex_wait_requeue_pi()
887 			 * and return -EWOULDBLOCK directly.  in futex_wait_requeue_pi()
889 			ret = -EWOULDBLOCK;  in futex_wait_requeue_pi()
898 		hrtimer_cancel(&to->timer);  in futex_wait_requeue_pi()
899 		destroy_hrtimer_on_stack(&to->timer);  in futex_wait_requeue_pi()