Lines Matching refs:futex
6 * Generalized futexes, futex requeueing, misc fixes by Ingo Molnar
12 * Robust futex support started by Ingo Molnar
16 * PI-futex support started by Ingo Molnar and Thomas Gleixner
51 #include <vdso/futex.h>
55 #include "futex.h"
201 struct futex_mm_phash *mmph = &mm->futex.phash;
226 scoped_guard(mutex, &mm->futex.phash.lock) {
229 fph = mm->futex.phash.hash_new;
231 mm->futex.phash.hash_new = NULL;
253 fph = rcu_dereference(mm->futex.phash.hash);
360 * @key: Pointer to the futex key for which the hash is calculated
380 fph = rcu_dereference(key->private.mm->futex.phash.hash);
415 * @flags: futex flags
446 * the file while it has a (shared) futex waiting on it. This mapping will have
480 * get_futex_key() - Get parameters which are the keys for a futex
481 * @uaddr: virtual address of the futex
501 * This allows (cross process, where applicable) identification of the futex
524 * The futex address must be "naturally" aligned.
592 /* Ignore any VERIFY_READ mapping (futex common case) */
667 * If the futex key is stored in anonymous memory, then the associated
677 * sense for futex operations.
692 * The associated futex object in this case is the inode and
756 * futex_top_waiter() - Return the highest priority waiter on a futex
758 * @key: the futex key (to distinguish it from other futex futex_q's)
775 * @ret: owner's current futex lock status
790 mutex_lock(&exiting->futex.exit_mutex);
797 * through the futex maze.
799 mutex_unlock(&exiting->futex.exit_mutex);
969 * Process a futex-list entry, check whether it's owned by the
991 * the owner of the futex (by the TID check below).
995 * 1. The unlock path releases the user space futex value and
996 * before it can execute the futex() syscall to wake up
1000 * futex in user space.
1003 * thread has acquired the futex, but before it can set
1012 * i. An ordinary futex wakeup after unlock (with or
1017 * As a result, the futex world will be in one of four states:
1019 * A. The futex word is 0 (unlocked)
1020 * B. The futex word is owned by another thread
1022 * C. The futex word is owned by another thread
1024 * D. The futex's owner died and OWNER_DIED is set
1030 * waiter dies before acquiring the futex (or setting the
1032 * waiter down the line to uphold the futex invariants and
1039 * here. Our thread is *not* the owner of the futex.
1046 * 2) The futex word is in one of the states A, B or D
1047 * 3) Regular futex: @pi == false
1050 * portion of the futex word differs from our thread's TID
1067 * Ok, this dying thread is truly holding a futex
1082 * access fails we try to fault in the futex with R/W
1085 * give up and leave the futex locked.
1138 * Walk curr->futex.robust_list (very carefully, it's a userspace list!)
1145 struct robust_list_head __user *head = curr->futex.robust_list;
1158 * Fetch the relative futex offset:
1206 struct robust_list_head __user *head = current->futex.robust_list;
1221 current->futex.robust_list = NULL;
1252 * Walk curr->futex.robust_list (very carefully, it's a userspace list!)
1259 struct compat_robust_list_head __user *head = current->futex.compat_robust_list;
1273 * Fetch the relative futex offset:
1324 struct compat_robust_list_head __user *head = current->futex.compat_robust_list;
1331 current->futex.compat_robust_list = NULL;
1343 * (Robust-futex cleanup is separate and might save the day for userspace.)
1347 struct list_head *next, *head = &curr->futex.pi_state_list;
1461 if (unlikely(tsk->futex.robust_list)) {
1463 tsk->futex.robust_list = NULL;
1467 if (unlikely(tsk->futex.compat_robust_list)) {
1469 tsk->futex.compat_robust_list = NULL;
1473 if (unlikely(!list_empty(&tsk->futex.pi_state_list)))
1478 * futex_exit_recursive - Set the tasks futex state to FUTEX_STATE_DEAD
1481 * Set the futex exit state of the task lockless. The futex waiter code
1483 * actually finished the futex cleanup. The worst case for this is that the
1488 * This is best effort. Either the futex exit code has run already or
1489 * not. If the OWNER_DIED bit has been set on the futex then the waiter can
1491 * futex exit code did not run yet, then an already queued waiter might
1497 if (tsk->futex.state == FUTEX_STATE_EXITING) {
1498 __assume_ctx_lock(&tsk->futex.exit_mutex);
1499 mutex_unlock(&tsk->futex.exit_mutex);
1501 tsk->futex.state = FUTEX_STATE_DEAD;
1505 __acquires(&tsk->futex.exit_mutex)
1510 * tsk->futex.exit_mutex when it observes FUTEX_STATE_EXITING in
1513 mutex_lock(&tsk->futex.exit_mutex);
1527 tsk->futex.state = FUTEX_STATE_EXITING;
1532 __releases(&tsk->futex.exit_mutex)
1535 tsk->futex.state = FUTEX_STATE_DEAD;
1541 mutex_unlock(&tsk->futex.exit_mutex);
1549 * same, but from a futex perspective the task has to be treated like an exiting
1573 * This store does not have to take tsk::futex::exit_mutex because the
1583 tsk->futex.state = FUTEX_STATE_OK;
1598 * futex-ref
1619 WARN_ON_ONCE(atomic_long_read(&mm->futex.phash.atomic) != 0);
1626 atomic_long_set(&mm->futex.phash.atomic, LONG_MAX);
1629 call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu);
1650 unsigned int *ptr = per_cpu_ptr(mm->futex.phash.ref, cpu);
1658 this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */
1666 ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex.phash.atomic);
1676 struct mm_struct *mm = container_of(head, struct mm_struct, futex.phash.rcu);
1677 struct futex_private_hash *fph = rcu_dereference_raw(mm->futex.phash.hash);
1706 WARN_ON_ONCE(rcu_dereference_raw(mm->futex.phash.hash) != fph);
1717 * guard(rcu); guard(mm->futex.phash.lock);
1718 * fph = mm->futex.phash.hash;
1719 * rcu_assign_pointer(&mm->futex.phash.hash, new);
1734 if (poll_state_synchronize_rcu(mm->futex.phash.batches)) {
1742 call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu);
1752 __this_cpu_inc(*mm->futex.phash.ref);
1756 return atomic_long_inc_not_zero(&mm->futex.phash.atomic);
1766 __this_cpu_dec(*mm->futex.phash.ref);
1770 return atomic_long_dec_and_test(&mm->futex.phash.atomic);
1782 return atomic_long_read(&mm->futex.phash.atomic) == 0;
1796 free_percpu(mm->futex.phash.ref);
1797 kvfree(mm->futex.phash.hash_new);
1798 fph = rcu_dereference_raw(mm->futex.phash.hash);
1804 struct futex_mm_phash *mmph = &mm->futex.phash;
1854 fph = rcu_dereference(mm->futex.phash.hash);
1862 if (!mm->futex.phash.ref) {
1873 if (cmpxchg(&mm->futex.phash.ref, NULL, ref))
1921 scoped_guard(mutex, &mm->futex.phash.lock) {
1925 cur = rcu_dereference_protected(mm->futex.phash.hash,
1926 lockdep_is_held(&mm->futex.phash.lock));
1927 new = mm->futex.phash.hash_new;
1928 mm->futex.phash.hash_new = NULL;
1938 mm->futex.phash.hash_new = new;
1968 * Will set mm->futex.phash.new_hash on failure;
1989 fph = rcu_dereference(current->mm->futex.phash.hash);
2016 fph = rcu_dereference(current->mm->futex.phash.hash);
2059 futex_hash_init_mm(&mm->futex);
2060 futex_robust_unlock_init_mm(&mm->futex);
2133 pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n",