1 // SPDX-License-Identifier: GPL-2.0-or-later
17 	if (likely(current->futex.pi_state_cache))  in refill_pi_state_cache()
23 		return -ENOMEM;  in refill_pi_state_cache()
25 	INIT_LIST_HEAD(&pi_state->list);  in refill_pi_state_cache()
27 	pi_state->owner = NULL;  in refill_pi_state_cache()
28 	refcount_set(&pi_state->refcount, 1);  in refill_pi_state_cache()
29 	pi_state->key = FUTEX_KEY_INIT;  in refill_pi_state_cache()
31 	current->futex.pi_state_cache = pi_state;  in refill_pi_state_cache()
38 	struct futex_pi_state *pi_state = current->futex.pi_state_cache;  in alloc_pi_state()
41 	current->futex.pi_state_cache = NULL;  in alloc_pi_state()
49 	struct task_struct *old_owner = pi_state->owner;  in pi_state_update_owner()
51 	lockdep_assert_held(&pi_state->pi_mutex.wait_lock);  in pi_state_update_owner()
54 		raw_spin_lock(&old_owner->pi_lock);  in pi_state_update_owner()
55 		WARN_ON(list_empty(&pi_state->list));  in pi_state_update_owner()
56 		list_del_init(&pi_state->list);  in pi_state_update_owner()
57 		raw_spin_unlock(&old_owner->pi_lock);  in pi_state_update_owner()
61 		raw_spin_lock(&new_owner->pi_lock);  in pi_state_update_owner()
62 		WARN_ON(!list_empty(&pi_state->list));  in pi_state_update_owner()
63 		list_add(&pi_state->list, &new_owner->futex.pi_state_list);  in pi_state_update_owner()
64 		pi_state->owner = new_owner;  in pi_state_update_owner()
65 		raw_spin_unlock(&new_owner->pi_lock);  in pi_state_update_owner()
71 	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));  in get_pi_state()
83 	if (!refcount_dec_and_test(&pi_state->refcount))  in put_pi_state()
87 	 * If pi_state->owner is NULL, the owner is most probably dying  in put_pi_state()
90 	if (pi_state->owner) {  in put_pi_state()
93 		raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
95 		rt_mutex_proxy_unlock(&pi_state->pi_mutex);  in put_pi_state()
96 		raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
99 	if (current->futex.pi_state_cache) {  in put_pi_state()
103 		 * pi_state->list is already empty.  in put_pi_state()
104 		 * clear pi_state->owner.  in put_pi_state()
105 		 * refcount is at 0 - put it back to 1.  in put_pi_state()
107 		pi_state->owner = NULL;  in put_pi_state()
108 		refcount_set(&pi_state->refcount, 1);  in put_pi_state()
109 		current->futex.pi_state_cache = pi_state;  in put_pi_state()
116  *      Waiter | pi_state | pi->owner | uTID      | uODIED | ?
118  * [1]  NULL   | ---      | ---       | 0         | 0/1    | Valid
119  * [2]  NULL   | ---      | ---       | >0        | 0/1    | Valid
121  * [3]  Found  | NULL     | --        | Any       | 0/1    | Invalid
151  * [7]	pi_state->owner can only be NULL when the OWNER_DIED bit is set.
166  * hb->lock:
168  *	hb -> futex_q, relation
169  *	futex_q -> pi_state, relation
174  * pi_mutex->wait_lock:
180  * p->pi_lock:
182  *	p->futex.pi_state_list -> pi_state->list, relation
183  *	pi_mutex->owner -> pi_state->owner, relation
185  * pi_state->refcount:
192  *   hb->lock
193  *     pi_mutex->wait_lock
194  *       p->pi_lock
198  * The kernel tracks the task state in p::futex::state to protect against exit()
199  * and exec(). The states are:
201  * - FUTEX_STATE_OK when the task is alive and waiters can be attached
203  * - FUTEX_STATE_EXITING when the task cleans up the robust list and PI
205  *   cleanup is finished to re-evaluate the potential changes caused by the
208  * - FUTEX_STATE_DEAD when the task has cleaned up the robust list. This state
209  *   is set independent of exit() or exec(). In the exit() case the task is
210  *   gone. In the exec() case this ensures that nothing can attach to the task
217  *   FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD
219  * The valid state transitions for exec():
221  *   FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD -> FUTEX_STATE_OK
228  *    protect against a concurrent exit/exec cleanup by the owner. If the state
247  *    re-evaluate the situation.
264 	 * Userspace might have messed up non-PI and PI futexes [3]  in attach_to_pi_state()
267 		return -EINVAL;  in attach_to_pi_state()
270 	 * We get here with hb->lock held, and having found a  in attach_to_pi_state()
272 	 * has dropped the hb->lock in between futex_queue() and futex_unqueue_pi(),  in attach_to_pi_state()
281 	WARN_ON(!refcount_read(&pi_state->refcount));  in attach_to_pi_state()
287 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
308 		 * pi_state->rt_mutex will fixup owner.  in attach_to_pi_state()
310 		if (!pi_state->owner) {  in attach_to_pi_state()
338 		if (!pi_state->owner)  in attach_to_pi_state()
347 	if (pid != task_pid_vnr(pi_state->owner))  in attach_to_pi_state()
352 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
357 	ret = -EINVAL;  in attach_to_pi_state()
361 	ret = -EAGAIN;  in attach_to_pi_state()
365 	ret = -EFAULT;  in attach_to_pi_state()
369 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
386 	 *    tsk->flags |= PF_EXITING;	    *uaddr == 0x00000PID  in handle_exit_race()
391 	 *   }				     if (!tsk->flags & PF_EXITING) {  in handle_exit_race()
393 	 *  tsk->futex.state =               } else {  in handle_exit_race()
394 	 *	FUTEX_STATE_DEAD;              if (tsk->futex.state !=  in handle_exit_race()
396 	 *				         return -EAGAIN;  in handle_exit_race()
397 	 *				       return -ESRCH; <--- FAIL  in handle_exit_race()
407 		return -EFAULT;  in handle_exit_race()
411 		return -EAGAIN;  in handle_exit_race()
418 	return -ESRCH;  in handle_exit_race()
427 	 * This creates pi_state, we have hb->lock held, this means nothing can  in __attach_to_pi_owner()
436 	__assume_ctx_lock(&pi_state->pi_mutex.wait_lock);  in __attach_to_pi_owner()
437 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);  in __attach_to_pi_owner()
440 	pi_state->key = *key;  in __attach_to_pi_owner()
442 	WARN_ON(!list_empty(&pi_state->list));  in __attach_to_pi_owner()
443 	list_add(&pi_state->list, &p->futex.pi_state_list);  in __attach_to_pi_owner()
445 	 * Assignment without holding pi_state->pi_mutex.wait_lock is safe  in __attach_to_pi_owner()
448 	pi_state->owner = p;  in __attach_to_pi_owner()
464 	 * We are the first waiter - try to look up the real owner and attach  in attach_to_pi_owner()
471 		return -EAGAIN;  in attach_to_pi_owner()
476 	if (unlikely(p->flags & PF_KTHREAD)) {  in attach_to_pi_owner()
478 		return -EPERM;  in attach_to_pi_owner()
483 	 * exiting. To protect against the change of the task state from  in attach_to_pi_owner()
485 	 * required to do this protected by p->pi_lock, which prevents the owner  in attach_to_pi_owner()
489 	 * is attached to protect against a concurrent exit()/exec().  in attach_to_pi_owner()
491 	raw_spin_lock_irq(&p->pi_lock);  in attach_to_pi_owner()
494 	if (unlikely(p->futex.state != FUTEX_STATE_OK)) {  in attach_to_pi_owner()
503 		 * re-evaluates the situation.  in attach_to_pi_owner()
505 		if (p->futex.state == FUTEX_STATE_EXITING) {  in attach_to_pi_owner()
506 			raw_spin_unlock_irq(&p->pi_lock);  in attach_to_pi_owner()
508 			return -EBUSY;  in attach_to_pi_owner()
513 		raw_spin_unlock_irq(&p->pi_lock);  in attach_to_pi_owner()
526 		if (unlikely(p->mm != key->private.mm)) {  in attach_to_pi_owner()
527 			raw_spin_unlock_irq(&p->pi_lock);  in attach_to_pi_owner()
529 			return -EPERM;  in attach_to_pi_owner()
534 	raw_spin_unlock_irq(&p->pi_lock);  in attach_to_pi_owner()
547 		return -EFAULT;  in lock_pi_update_atomic()
554 	return curval != uval ? -EAGAIN : 0;  in lock_pi_update_atomic()
558  * futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
571  *  -  0 - ready to wait;
572  *  -  1 - acquired the lock;
573  *  - <0 - error
575  * The hb->lock must be held by the caller.
577  * @exiting is only set when the return value is -EBUSY. If so, this holds
597 		return -EFAULT;  in futex_lock_pi_atomic()
600 		return -EFAULT;  in futex_lock_pi_atomic()
606 		return -EDEADLK;  in futex_lock_pi_atomic()
609 		return -EDEADLK;  in futex_lock_pi_atomic()
617 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
653 			raw_spin_lock_irq(&task->pi_lock);  in futex_lock_pi_atomic()
655 			raw_spin_unlock_irq(&task->pi_lock);  in futex_lock_pi_atomic()
663 	 * the kernel and blocked on hb->lock.  in futex_lock_pi_atomic()
683 	__must_hold(&pi_state->pi_mutex.wait_lock)  in wake_futex_pi()
684 	__releases(&pi_state->pi_mutex.wait_lock)  in wake_futex_pi()
692 	new_owner = top_waiter->task;  in wake_futex_pi()
702 		ret = -EFAULT;  in wake_futex_pi()
710 		 * try the TID->0 transition) raced with a waiter setting the  in wake_futex_pi()
715 			ret = -EAGAIN;  in wake_futex_pi()
717 			ret = -EINVAL;  in wake_futex_pi()
727 		postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh);  in wake_futex_pi()
731 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
741 	__must_hold(&q->pi_state->pi_mutex.wait_lock)  in __fixup_pi_state_owner()
742 	__must_hold(q->lock_ptr)  in __fixup_pi_state_owner()
744 	struct futex_pi_state *pi_state = q->pi_state;  in __fixup_pi_state_owner()
749 	oldowner = pi_state->owner;  in __fixup_pi_state_owner()
754 	 *  - we stole the lock and pi_state->owner needs updating to reflect  in __fixup_pi_state_owner()
759 	 *  - someone stole our lock and we need to fix things to point to the  in __fixup_pi_state_owner()
784 		if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {  in __fixup_pi_state_owner()
793 		newowner = rt_mutex_owner(&pi_state->pi_mutex);  in __fixup_pi_state_owner()
803 			err = -EAGAIN;  in __fixup_pi_state_owner()
820 	if (!pi_state->owner)  in __fixup_pi_state_owner()
856 	 * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely  in __fixup_pi_state_owner()
857 	 * drop hb->lock since the caller owns the hb -> futex_q relation.  in __fixup_pi_state_owner()
858 	 * Dropping the pi_mutex->wait_lock requires the state revalidate.  in __fixup_pi_state_owner()
861 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in __fixup_pi_state_owner()
862 	spin_unlock(q->lock_ptr);  in __fixup_pi_state_owner()
865 	case -EFAULT:  in __fixup_pi_state_owner()
869 	case -EAGAIN:  in __fixup_pi_state_owner()
880 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in __fixup_pi_state_owner()
885 	if (pi_state->owner != oldowner)  in __fixup_pi_state_owner()
888 	/* Retry if err was -EAGAIN or the fault in succeeded */  in __fixup_pi_state_owner()
905 	 * The rtmutex has an owner - either current or some other  in __fixup_pi_state_owner()
908 	pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex));  in __fixup_pi_state_owner()
916 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()
919 	lockdep_assert_held(q->lock_ptr);  in fixup_pi_state_owner()
921 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
923 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
928  * fixup_pi_owner() - Post lock pi_state and corner case management
938  *  -  1 - success, lock taken;
939  *  -  0 - success, lock not taken;
940  *  - <0 - on error (-EFAULT)
947 		 * did a lock-steal - fix up the PI-state in that case:  in fixup_pi_owner()
949 		 * Speculative pi_state->owner read (we don't hold wait_lock);  in fixup_pi_owner()
950 		 * since we own the lock pi_state->owner == current is the  in fixup_pi_owner()
953 		if (q->pi_state->owner != current)  in fixup_pi_owner()
963 	 * Another speculative read; pi_state->owner == current is unstable  in fixup_pi_owner()
966 	if (q->pi_state->owner == current)  in fixup_pi_owner()
973 	if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current))  in fixup_pi_owner()
980  * Userspace tried a 0 -> TID atomic transition of the futex value
983  * on rt-mutexes, it does PI, etc. (Due to races the kernel might see
998 		return -ENOSYS;  in futex_lock_pi()
1001 		return -ENOMEM;  in futex_lock_pi()
1030 			case -EFAULT:  in futex_lock_pi()
1032 			case -EBUSY:  in futex_lock_pi()
1033 			case -EAGAIN:  in futex_lock_pi()
1036 				 * - EBUSY: Task is exiting and we just wait for the  in futex_lock_pi()
1038 				 * - EAGAIN: The user space value changed.  in futex_lock_pi()
1063 			ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
1065 			ret = ret ? 0 : -EWOULDBLOCK;  in futex_lock_pi()
1070 		 * Caution; releasing @hb in-scope. The hb->lock is still locked  in futex_lock_pi()
1075 		 * hb->lock during the requeue.  in futex_lock_pi()
1082 		 * On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not  in futex_lock_pi()
1084 		 * include hb->lock in the blocking chain, even through we'll not in  in futex_lock_pi()
1085 		 * fact hold it while blocking. This will lead it to report -EDEADLK  in futex_lock_pi()
1088 		 * Therefore acquire wait_lock while holding hb->lock, but drop the  in futex_lock_pi()
1090 		 * interleaves with futex_unlock_pi() -- which does a similar lock  in futex_lock_pi()
1091 		 * handoff -- such that the latter can observe the futex_q::pi_state  in futex_lock_pi()
1094 		raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
1101 		ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);  in futex_lock_pi()
1102 		raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);  in futex_lock_pi()
1113 		ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);  in futex_lock_pi()
1118 		 * unwind the above, however we canont lock hb->lock because  in futex_lock_pi()
1119 		 * rt_mutex already has a waiter enqueued and hb->lock can itself try  in futex_lock_pi()
1122 		 * Doing the cleanup without holding hb->lock can cause inconsistent  in futex_lock_pi()
1134 		if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))  in futex_lock_pi()
1151 		 * the lock, clear our -ETIMEDOUT or -EINTR.  in futex_lock_pi()
1156 		__release(&hb->lock);  in futex_lock_pi()
1185 		hrtimer_cancel(&to->timer);  in futex_lock_pi()
1186 		destroy_hrtimer_on_stack(&to->timer);  in futex_lock_pi()
1188 	return ret != -EINTR ? ret : -ERESTARTNOINTR;  in futex_lock_pi()
1192  * Userspace attempted a TID -> 0 atomic transition, and failed.
1193  * This is the in-kernel slowpath: we look up the PI state (if any),
1194  * and do the rt-mutex unlock.
1204 		return -ENOSYS;  in __futex_unlock_pi()
1207 		return -EFAULT;  in __futex_unlock_pi()
1212 		return -EPERM;  in __futex_unlock_pi()
1220 	spin_lock(&hb->lock);  in __futex_unlock_pi()
1230 		struct futex_pi_state *pi_state = top_waiter->pi_state;  in __futex_unlock_pi()
1233 		ret = -EINVAL;  in __futex_unlock_pi()
1241 		if (pi_state->owner != current)  in __futex_unlock_pi()
1245 		 * By taking wait_lock while still holding hb->lock, we ensure  in __futex_unlock_pi()
1250 		 * rt_waiter without holding hb->lock, it is possible for  in __futex_unlock_pi()
1259 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in __futex_unlock_pi()
1262 		 * Futex vs rt_mutex waiter state -- if there are no rt_mutex  in __futex_unlock_pi()
1265 		 * new futex_lock_pi() is not using this stale PI-state while  in __futex_unlock_pi()
1270 		rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);  in __futex_unlock_pi()
1277 				top_waiter->drop_fph = futex_private_hash(key.private.mm);  in __futex_unlock_pi()
1280 			raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in __futex_unlock_pi()
1285 		spin_unlock(&hb->lock);  in __futex_unlock_pi()
1287 		/* drops pi_state->pi_mutex.wait_lock */  in __futex_unlock_pi()
1299 		 * pagefault, so retry the user-access and the wakeup:  in __futex_unlock_pi()
1301 		if (ret == -EFAULT)  in __futex_unlock_pi()
1307 		if (ret == -EAGAIN)  in __futex_unlock_pi()
1319 	 * on hb->lock. So we can safely ignore them. We do neither  in __futex_unlock_pi()
1324 		spin_unlock(&hb->lock);  in __futex_unlock_pi()
1326 		case -EFAULT:  in __futex_unlock_pi()
1329 		case -EAGAIN:  in __futex_unlock_pi()
1341 	ret = (curval == uval) ? 0 : -EAGAIN;  in __futex_unlock_pi()
1344 	spin_unlock(&hb->lock);  in __futex_unlock_pi()
1368 		return -EFAULT;  in futex_unlock_pi()