Lines Matching +full:lock +full:- +full:state
1 // SPDX-License-Identifier: GPL-2.0-or-later
17 if (likely(current->pi_state_cache))
23 return -ENOMEM;
25 INIT_LIST_HEAD(&pi_state->list);
27 pi_state->owner = NULL;
28 refcount_set(&pi_state->refcount, 1);
29 pi_state->key = FUTEX_KEY_INIT;
31 current->pi_state_cache = pi_state;
38 struct futex_pi_state *pi_state = current->pi_state_cache;
41 current->pi_state_cache = NULL;
49 struct task_struct *old_owner = pi_state->owner;
51 lockdep_assert_held(&pi_state->pi_mutex.wait_lock);
54 raw_spin_lock(&old_owner->pi_lock);
55 WARN_ON(list_empty(&pi_state->list));
56 list_del_init(&pi_state->list);
57 raw_spin_unlock(&old_owner->pi_lock);
61 raw_spin_lock(&new_owner->pi_lock);
62 WARN_ON(!list_empty(&pi_state->list));
63 list_add(&pi_state->list, &new_owner->pi_state_list);
64 pi_state->owner = new_owner;
65 raw_spin_unlock(&new_owner->pi_lock);
71 WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));
83 if (!refcount_dec_and_test(&pi_state->refcount))
87 * If pi_state->owner is NULL, the owner is most probably dying
90 if (pi_state->owner) {
93 raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);
95 rt_mutex_proxy_unlock(&pi_state->pi_mutex);
96 raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);
99 if (current->pi_state_cache) {
103 * pi_state->list is already empty.
104 * clear pi_state->owner.
105 * refcount is at 0 - put it back to 1.
107 pi_state->owner = NULL;
108 refcount_set(&pi_state->refcount, 1);
109 current->pi_state_cache = pi_state;
116 * Waiter | pi_state | pi->owner | uTID | uODIED | ?
118 * [1] NULL | --- | --- | 0 | 0/1 | Valid
119 * [2] NULL | --- | --- | >0 | 0/1 | Valid
121 * [3] Found | NULL | -- | Any | 0/1 | Invalid
142 * [4] Valid state after exit_robust_list(), which sets the user space
148 * [6] Valid state after exit_pi_state_list() which sets the new owner in
151 * [7] pi_state->owner can only be NULL when the OWNER_DIED bit is set.
155 * [9] There is no transient state which sets the user space TID to 0
159 * [10] There is no transient state which leaves owner and user space
166 * hb->lock:
168 * hb -> futex_q, relation
169 * futex_q -> pi_state, relation
174 * pi_mutex->wait_lock:
180 * p->pi_lock:
182 * p->pi_state_list -> pi_state->list, relation
183 * pi_mutex->owner -> pi_state->owner, relation
185 * pi_state->refcount:
190 * Lock order:
192 * hb->lock
193 * pi_mutex->wait_lock
194 * p->pi_lock
212 * Userspace might have messed up non-PI and PI futexes [3]
215 return -EINVAL;
218 * We get here with hb->lock held, and having found a
220 * has dropped the hb->lock in between futex_queue() and futex_unqueue_pi(),
229 WARN_ON(!refcount_read(&pi_state->refcount));
233 * and do the state validation.
235 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
256 * pi_state->rt_mutex will fixup owner.
258 if (!pi_state->owner) {
260 * No pi state owner, but the user space TID
261 * is not 0. Inconsistent state. [5]
266 * Take a ref on the state and return success. [4]
274 * acquired the rtmutex in the pi state, but did not
277 * Take a ref on the state and return success. [6]
286 if (!pi_state->owner)
292 * state exists then the owner TID must be the same as the
295 if (pid != task_pid_vnr(pi_state->owner))
300 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
305 ret = -EINVAL;
309 ret = -EAGAIN;
313 ret = -EFAULT;
317 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
327 * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the
330 if (tsk && tsk->futex_state != FUTEX_STATE_DEAD)
331 return -EBUSY;
342 * tsk->flags |= PF_EXITING; *uaddr == 0x00000PID
347 * } if (!tsk->flags & PF_EXITING) {
349 * tsk->futex_state = } else {
350 * FUTEX_STATE_DEAD; if (tsk->futex_state !=
352 * return -EAGAIN;
353 * return -ESRCH; <--- FAIL
363 return -EFAULT;
367 return -EAGAIN;
374 return -ESRCH;
381 * No existing pi state. First waiter. [2]
383 * This creates pi_state, we have hb->lock held, this means nothing can
384 * observe this state, wait_lock is irrelevant.
389 * Initialize the pi_mutex in locked state and make @p
392 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
395 pi_state->key = *key;
397 WARN_ON(!list_empty(&pi_state->list));
398 list_add(&pi_state->list, &p->pi_state_list);
400 * Assignment without holding pi_state->pi_mutex.wait_lock is safe
403 pi_state->owner = p;
419 * We are the first waiter - try to look up the real owner and attach
426 return -EAGAIN;
431 if (unlikely(p->flags & PF_KTHREAD)) {
433 return -EPERM;
437 * We need to look at the task state to figure out, whether the
438 * task is exiting. To protect against the change of the task state
439 * in futex_exit_release(), we do this protected by p->pi_lock:
441 raw_spin_lock_irq(&p->pi_lock);
442 if (unlikely(p->futex_state != FUTEX_STATE_OK)) {
444 * The task is on the way out. When the futex state is
450 raw_spin_unlock_irq(&p->pi_lock);
457 * required to prevent a live lock when the current task
460 if (ret == -EBUSY)
468 raw_spin_unlock_irq(&p->pi_lock);
481 return -EFAULT;
488 return curval != uval ? -EAGAIN : 0;
492 * futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
498 * @task: the task to perform the atomic lock work for. This will
505 * - 0 - ready to wait;
506 * - 1 - acquired the lock;
507 * - <0 - error
509 * The hb->lock must be held by the caller.
511 * @exiting is only set when the return value is -EBUSY. If so, this holds
531 return -EFAULT;
534 return -EFAULT;
540 return -EDEADLK;
543 return -EDEADLK;
546 * Lookup existing state first. If it exists, try to attach to
551 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
577 * state attached to the new owner of the user space futex.
587 raw_spin_lock_irq(&task->pi_lock);
589 raw_spin_unlock_irq(&task->pi_lock);
597 * the kernel and blocked on hb->lock.
624 new_owner = top_waiter->task;
628 * enabled while there is PI state around. We cleanup the owner
634 ret = -EFAULT;
642 * try the TID->0 transition) raced with a waiter setting the
644 * bucket lock, retry the operation.
647 ret = -EAGAIN;
649 ret = -EINVAL;
659 postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh);
663 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
674 struct futex_pi_state *pi_state = q->pi_state;
679 oldowner = pi_state->owner;
684 * - we stole the lock and pi_state->owner needs updating to reflect
689 * - someone stole our lock and we need to fix things to point to the
700 * pi_state in an inconsistent state when we fault here, because we
702 * in the PID checks when attaching to PI state .
714 if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {
715 /* We got the lock. pi_state is correct. Tell caller. */
723 newowner = rt_mutex_owner(&pi_state->pi_mutex);
727 * that state because it's inconsistent vs. the user space
728 * state. So drop the locks and try again. It's a valid
733 err = -EAGAIN;
750 if (!pi_state->owner)
786 * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely
787 * drop hb->lock since the caller owns the hb -> futex_q relation.
788 * Dropping the pi_mutex->wait_lock requires the state revalidate.
791 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
792 spin_unlock(q->lock_ptr);
795 case -EFAULT:
799 case -EAGAIN:
810 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
815 if (pi_state->owner != oldowner)
818 /* Retry if err was -EAGAIN or the fault in succeeded */
823 * fault_in_user_writeable() failed so user state is immutable. At
824 * best we can make the kernel state consistent but user state will
832 * to guarantee consistent state. Keep it simple. Userspace asked
833 * for this wreckaged state.
835 * The rtmutex has an owner - either current or some other
838 pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex));
846 struct futex_pi_state *pi_state = q->pi_state;
849 lockdep_assert_held(q->lock_ptr);
851 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
853 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
858 * fixup_pi_owner() - Post lock pi_state and corner case management
863 * After attempting to lock an rt_mutex, this function is called to cleanup
865 * acquire the lock. Must be called with the hb lock held.
868 * - 1 - success, lock taken;
869 * - 0 - success, lock not taken;
870 * - <0 - on error (-EFAULT)
876 * Got the lock. We might not be the anticipated owner if we
877 * did a lock-steal - fix up the PI-state in that case:
879 * Speculative pi_state->owner read (we don't hold wait_lock);
880 * since we own the lock pi_state->owner == current is the
881 * stable state, anything else needs more attention.
883 if (q->pi_state->owner != current)
889 * If we didn't get the lock; check if anybody stole it from us. In
893 * Another speculative read; pi_state->owner == current is unstable
896 if (q->pi_state->owner == current)
900 * Paranoia check. If we did not take the lock, then we should not be
901 * the owner of the rt_mutex. Warn and establish consistent state.
903 if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current))
910 * Userspace tried a 0 -> TID atomic transition of the futex value
913 * on rt-mutexes, it does PI, etc. (Due to races the kernel might see
928 return -ENOSYS;
931 return -ENOMEM;
950 * Atomic work succeeded and we got the lock,
955 /* We got the lock. */
958 case -EFAULT:
960 case -EBUSY:
961 case -EAGAIN:
964 * - EBUSY: Task is exiting and we just wait for the
966 * - EAGAIN: The user space value changed.
972 * this task might loop forever, aka. live lock.
990 ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
992 ret = ret ? 0 : -EWOULDBLOCK;
997 * Caution; releasing @hb in-scope. The hb->lock is still locked
1002 * the thread, performing resize, will block on hb->lock during
1008 * under the hb lock, but that *should* work because it does nothing.
1015 * On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not
1017 * include hb->lock in the blocking chain, even through we'll not in
1018 * fact hold it while blocking. This will lead it to report -EDEADLK
1021 * Therefore acquire wait_lock while holding hb->lock, but drop the
1023 * interleaves with futex_unlock_pi() -- which does a similar lock
1024 * handoff -- such that the latter can observe the futex_q::pi_state
1027 raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
1034 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);
1035 raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);
1046 ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
1050 * If we failed to acquire the lock (deadlock/signal/timeout), we must
1051 * unwind the above, however we canont lock hb->lock because
1052 * rt_mutex already has a waiter enqueued and hb->lock can itself try
1055 * Doing the cleanup without holding hb->lock can cause inconsistent
1056 * state between hb and pi_state, but only in the direction of not
1067 if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
1082 * Fixup the pi_state owner and possibly acquire the lock if we
1088 * the lock, clear our -ETIMEDOUT or -EINTR.
1121 hrtimer_cancel(&to->timer);
1122 destroy_hrtimer_on_stack(&to->timer);
1124 return ret != -EINTR ? ret : -ERESTARTNOINTR;
1128 * Userspace attempted a TID -> 0 atomic transition, and failed.
1129 * This is the in-kernel slowpath: we look up the PI state (if any),
1130 * and do the rt-mutex unlock.
1140 return -ENOSYS;
1144 return -EFAULT;
1146 * We release only a lock we actually own:
1149 return -EPERM;
1156 spin_lock(&hb->lock);
1166 struct futex_pi_state *pi_state = top_waiter->pi_state;
1169 ret = -EINVAL;
1177 if (pi_state->owner != current)
1181 * By taking wait_lock while still holding hb->lock, we ensure
1186 * rt_waiter without holding hb->lock, it is possible for
1195 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
1198 * Futex vs rt_mutex waiter state -- if there are no rt_mutex
1201 * new futex_lock_pi() is not using this stale PI-state while
1206 rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
1213 top_waiter->drop_hb_ref = true;
1215 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1220 spin_unlock(&hb->lock);
1222 /* drops pi_state->pi_mutex.wait_lock */
1234 * pagefault, so retry the user-access and the wakeup:
1236 if (ret == -EFAULT)
1242 if (ret == -EAGAIN)
1245 * wake_futex_pi has detected invalid state. Tell user
1252 * We have no kernel internal state, i.e. no waiters in the
1254 * on hb->lock. So we can safely ignore them. We do neither
1259 spin_unlock(&hb->lock);
1261 case -EFAULT:
1264 case -EAGAIN:
1276 ret = (curval == uval) ? 0 : -EAGAIN;
1279 spin_unlock(&hb->lock);