127 __futex_hash(union futex_key *key, struct futex_private_hash *fph);
130 static bool futex_ref_get(struct futex_private_hash *fph);
131 static bool futex_ref_put(struct futex_private_hash *fph);
132 static bool futex_ref_is_dead(struct futex_private_hash *fph);
145 static bool futex_private_hash_get(struct futex_private_hash *fph)  in futex_private_hash_get()  argument
147 	return futex_ref_get(fph);  in futex_private_hash_get()
150 void futex_private_hash_put(struct futex_private_hash *fph)  in futex_private_hash_put()  argument
152 	if (futex_ref_put(fph))  in futex_private_hash_put()
153 		wake_up_var(fph->mm);  in futex_private_hash_put()
165 	struct futex_private_hash *fph = hb->priv;  in futex_hash_get()  local
167 	if (!fph)  in futex_hash_get()
169 	WARN_ON_ONCE(!futex_private_hash_get(fph));  in futex_hash_get()
174 	struct futex_private_hash *fph = hb->priv;  in futex_hash_put()  local
176 	if (!fph)  in futex_hash_put()
178 	futex_private_hash_put(fph);  in futex_hash_put()
182 __futex_hash_private(union futex_key *key, struct futex_private_hash *fph)  in __futex_hash_private()  argument
189 	if (!fph)  in __futex_hash_private()
190 		fph = rcu_dereference(key->private.mm->futex_phash);  in __futex_hash_private()
191 	if (!fph || !fph->hash_mask)  in __futex_hash_private()
197 	return &fph->queues[hash & fph->hash_mask];  in __futex_hash_private()
238 	struct futex_private_hash *fph;  in __futex_pivot_hash()  local
242 	fph = rcu_dereference_protected(mm->futex_phash,  in __futex_pivot_hash()
244 	if (fph) {  in __futex_pivot_hash()
245 		if (!futex_ref_is_dead(fph)) {  in __futex_pivot_hash()
250 		futex_rehash_private(fph, new);  in __futex_pivot_hash()
257 	kvfree_rcu(fph, rcu);  in __futex_pivot_hash()
264 		struct futex_private_hash *fph;  in futex_pivot_hash()  local
266 		fph = mm->futex_phash_new;  in futex_pivot_hash()
267 		if (fph) {  in futex_pivot_hash()
269 			__futex_pivot_hash(mm, fph);  in futex_pivot_hash()
289 		struct futex_private_hash *fph;  in futex_private_hash()  local
291 		fph = rcu_dereference(mm->futex_phash);  in futex_private_hash()
292 		if (!fph)  in futex_private_hash()
295 		if (futex_private_hash_get(fph))  in futex_private_hash()
296 			return fph;  in futex_private_hash()
304 	struct futex_private_hash *fph;  in futex_hash()  local
310 		fph = hb->priv;  in futex_hash()
312 		if (!fph || futex_private_hash_get(fph))  in futex_hash()
322 __futex_hash_private(union futex_key *key, struct futex_private_hash *fph)  in __futex_hash_private()  argument
415 __futex_hash(union futex_key *key, struct futex_private_hash *fph)  in __futex_hash()  argument
423 		hb = __futex_hash_private(key, fph);  in __futex_hash()
1517 				   struct futex_private_hash *fph)  in futex_hash_bucket_init()  argument
1520 	fhb->priv = fph;  in futex_hash_bucket_init()
1544 static void __futex_ref_atomic_begin(struct futex_private_hash *fph)  in __futex_ref_atomic_begin()  argument
1546 	struct mm_struct *mm = fph->mm;  in __futex_ref_atomic_begin()
1561 	smp_store_release(&fph->state, FR_ATOMIC);  in __futex_ref_atomic_begin()
1566 static void __futex_ref_atomic_end(struct futex_private_hash *fph)  in __futex_ref_atomic_end()  argument
1568 	struct mm_struct *mm = fph->mm;  in __futex_ref_atomic_end()
1578 	WARN_ON_ONCE(fph->state != FR_ATOMIC);  in __futex_ref_atomic_end()
1611 	struct futex_private_hash *fph = rcu_dereference_raw(mm->futex_phash);  in futex_ref_rcu()  local
1613 	if (fph->state == FR_PERCPU) {  in futex_ref_rcu()
1623 		__futex_ref_atomic_begin(fph);  in futex_ref_rcu()
1627 	__futex_ref_atomic_end(fph);  in futex_ref_rcu()
1633 static void futex_ref_drop(struct futex_private_hash *fph)  in futex_ref_drop()  argument
1635 	struct mm_struct *mm = fph->mm;  in futex_ref_drop()
1640 	WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) != fph);  in futex_ref_drop()
1672 		__futex_ref_atomic_begin(fph);  in futex_ref_drop()
1679 static bool futex_ref_get(struct futex_private_hash *fph)  in futex_ref_get()  argument
1681 	struct mm_struct *mm = fph->mm;  in futex_ref_get()
1685 	if (READ_ONCE(fph->state) == FR_PERCPU) {  in futex_ref_get()
1693 static bool futex_ref_put(struct futex_private_hash *fph)  in futex_ref_put()  argument
1695 	struct mm_struct *mm = fph->mm;  in futex_ref_put()
1699 	if (READ_ONCE(fph->state) == FR_PERCPU) {  in futex_ref_put()
1707 static bool futex_ref_is_dead(struct futex_private_hash *fph)  in futex_ref_is_dead()  argument
1709 	struct mm_struct *mm = fph->mm;  in futex_ref_is_dead()
1713 	if (smp_load_acquire(&fph->state) == FR_PERCPU)  in futex_ref_is_dead()
1733 	struct futex_private_hash *fph;  in futex_hash_free()  local
1737 	fph = rcu_dereference_raw(mm->futex_phash);  in futex_hash_free()
1738 	if (fph)  in futex_hash_free()
1739 		kvfree(fph);  in futex_hash_free()
1744 	struct futex_private_hash *fph;  in futex_pivot_pending()  local
1751 	fph = rcu_dereference(mm->futex_phash);  in futex_pivot_pending()
1752 	return futex_ref_is_dead(fph);  in futex_pivot_pending()
1782 	struct futex_private_hash *fph;  in futex_hash_allocate()  local
1793 		fph = rcu_dereference(mm->futex_phash);  in futex_hash_allocate()
1794 		if (fph && !fph->hash_mask) {  in futex_hash_allocate()
1812 	fph = kvzalloc(struct_size(fph, queues, hash_slots),  in futex_hash_allocate()
1814 	if (!fph)  in futex_hash_allocate()
1817 	fph->hash_mask = hash_slots ? hash_slots - 1 : 0;  in futex_hash_allocate()
1818 	fph->custom = custom;  in futex_hash_allocate()
1819 	fph->mm = mm;  in futex_hash_allocate()
1822 		futex_hash_bucket_init(&fph->queues[i], fph);  in futex_hash_allocate()
1841 		if (fph) {  in futex_hash_allocate()
1848 				free = fph;  in futex_hash_allocate()
1865 				if (futex_hash_less(new, fph)) {  in futex_hash_allocate()
1867 					new = fph;  in futex_hash_allocate()
1869 					free = fph;  in futex_hash_allocate()
1872 				new = fph;  in futex_hash_allocate()
1874 			fph = NULL;  in futex_hash_allocate()
1892 	struct futex_private_hash *fph;  in futex_hash_allocate_default()  local
1902 		fph = rcu_dereference(current->mm->futex_phash);  in futex_hash_allocate_default()
1903 		if (fph) {  in futex_hash_allocate_default()
1904 			if (fph->custom)  in futex_hash_allocate_default()
1907 			current_buckets = fph->hash_mask + 1;  in futex_hash_allocate_default()
1926 	struct futex_private_hash *fph;  in futex_hash_get_slots()  local
1929 	fph = rcu_dereference(current->mm->futex_phash);  in futex_hash_get_slots()
1930 	if (fph && fph->hash_mask)  in futex_hash_get_slots()
1931 		return fph->hash_mask + 1;  in futex_hash_get_slots()