1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _FUTEX_H 3 #define _FUTEX_H 4 5 #include <linux/futex.h> 6 #include <linux/sched/wake_q.h> 7 8 #ifdef CONFIG_PREEMPT_RT 9 #include <linux/rcuwait.h> 10 #endif 11 12 #include <asm/futex.h> 13 14 /* 15 * Futex flags used to encode options to functions and preserve them across 16 * restarts. 17 */ 18 #ifdef CONFIG_MMU 19 # define FLAGS_SHARED 0x01 20 #else 21 /* 22 * NOMMU does not have per process address space. Let the compiler optimize 23 * code away. 24 */ 25 # define FLAGS_SHARED 0x00 26 #endif 27 #define FLAGS_CLOCKRT 0x02 28 #define FLAGS_HAS_TIMEOUT 0x04 29 30 #ifdef CONFIG_FAIL_FUTEX 31 extern bool should_fail_futex(bool fshared); 32 #else 33 static inline bool should_fail_futex(bool fshared) 34 { 35 return false; 36 } 37 #endif 38 39 /* 40 * Hash buckets are shared by all the futex_keys that hash to the same 41 * location. Each key may have multiple futex_q structures, one for each task 42 * waiting on a futex. 43 */ 44 struct futex_hash_bucket { 45 atomic_t waiters; 46 spinlock_t lock; 47 struct plist_head chain; 48 } ____cacheline_aligned_in_smp; 49 50 /* 51 * Priority Inheritance state: 52 */ 53 struct futex_pi_state { 54 /* 55 * list of 'owned' pi_state instances - these have to be 56 * cleaned up in do_exit() if the task exits prematurely: 57 */ 58 struct list_head list; 59 60 /* 61 * The PI object: 62 */ 63 struct rt_mutex_base pi_mutex; 64 65 struct task_struct *owner; 66 refcount_t refcount; 67 68 union futex_key key; 69 } __randomize_layout; 70 71 /** 72 * struct futex_q - The hashed futex queue entry, one per waiting task 73 * @list: priority-sorted list of tasks waiting on this futex 74 * @task: the task waiting on the futex 75 * @lock_ptr: the hash bucket lock 76 * @key: the key the futex is hashed on 77 * @pi_state: optional priority inheritance state 78 * @rt_waiter: rt_waiter storage for use with requeue_pi 79 * @requeue_pi_key: the requeue_pi target futex key 80 * @bitset: bitset for the optional bitmasked wakeup 81 * @requeue_state: State field for futex_requeue_pi() 82 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) 83 * 84 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so 85 * we can wake only the relevant ones (hashed queues may be shared). 86 * 87 * A futex_q has a woken state, just like tasks have TASK_RUNNING. 88 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. 89 * The order of wakeup is always to make the first condition true, then 90 * the second. 91 * 92 * PI futexes are typically woken before they are removed from the hash list via 93 * the rt_mutex code. See futex_unqueue_pi(). 94 */ 95 struct futex_q { 96 struct plist_node list; 97 98 struct task_struct *task; 99 spinlock_t *lock_ptr; 100 union futex_key key; 101 struct futex_pi_state *pi_state; 102 struct rt_mutex_waiter *rt_waiter; 103 union futex_key *requeue_pi_key; 104 u32 bitset; 105 atomic_t requeue_state; 106 #ifdef CONFIG_PREEMPT_RT 107 struct rcuwait requeue_wait; 108 #endif 109 } __randomize_layout; 110 111 extern const struct futex_q futex_q_init; 112 113 enum futex_access { 114 FUTEX_READ, 115 FUTEX_WRITE 116 }; 117 118 extern int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, 119 enum futex_access rw); 120 121 extern struct hrtimer_sleeper * 122 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 123 int flags, u64 range_ns); 124 125 extern struct futex_hash_bucket *futex_hash(union futex_key *key); 126 127 /** 128 * futex_match - Check whether two futex keys are equal 129 * @key1: Pointer to key1 130 * @key2: Pointer to key2 131 * 132 * Return 1 if two futex_keys are equal, 0 otherwise. 133 */ 134 static inline int futex_match(union futex_key *key1, union futex_key *key2) 135 { 136 return (key1 && key2 137 && key1->both.word == key2->both.word 138 && key1->both.ptr == key2->both.ptr 139 && key1->both.offset == key2->both.offset); 140 } 141 142 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 143 struct futex_q *q, struct futex_hash_bucket **hb); 144 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, 145 struct hrtimer_sleeper *timeout); 146 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q); 147 148 extern int fault_in_user_writeable(u32 __user *uaddr); 149 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval); 150 extern int futex_get_value_locked(u32 *dest, u32 __user *from); 151 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key); 152 153 extern void __futex_unqueue(struct futex_q *q); 154 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb); 155 extern int futex_unqueue(struct futex_q *q); 156 157 /** 158 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket 159 * @q: The futex_q to enqueue 160 * @hb: The destination hash bucket 161 * 162 * The hb->lock must be held by the caller, and is released here. A call to 163 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The 164 * exceptions involve the PI related operations, which may use futex_unqueue_pi() 165 * or nothing if the unqueue is done as part of the wake process and the unqueue 166 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for 167 * an example). 168 */ 169 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb) 170 __releases(&hb->lock) 171 { 172 __futex_queue(q, hb); 173 spin_unlock(&hb->lock); 174 } 175 176 extern void futex_unqueue_pi(struct futex_q *q); 177 178 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting); 179 180 /* 181 * Reflects a new waiter being added to the waitqueue. 182 */ 183 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb) 184 { 185 #ifdef CONFIG_SMP 186 atomic_inc(&hb->waiters); 187 /* 188 * Full barrier (A), see the ordering comment above. 189 */ 190 smp_mb__after_atomic(); 191 #endif 192 } 193 194 /* 195 * Reflects a waiter being removed from the waitqueue by wakeup 196 * paths. 197 */ 198 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb) 199 { 200 #ifdef CONFIG_SMP 201 atomic_dec(&hb->waiters); 202 #endif 203 } 204 205 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb) 206 { 207 #ifdef CONFIG_SMP 208 /* 209 * Full barrier (B), see the ordering comment above. 210 */ 211 smp_mb(); 212 return atomic_read(&hb->waiters); 213 #else 214 return 1; 215 #endif 216 } 217 218 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q); 219 extern void futex_q_unlock(struct futex_hash_bucket *hb); 220 221 222 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb, 223 union futex_key *key, 224 struct futex_pi_state **ps, 225 struct task_struct *task, 226 struct task_struct **exiting, 227 int set_waiters); 228 229 extern int refill_pi_state_cache(void); 230 extern void get_pi_state(struct futex_pi_state *pi_state); 231 extern void put_pi_state(struct futex_pi_state *pi_state); 232 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked); 233 234 /* 235 * Express the locking dependencies for lockdep: 236 */ 237 static inline void 238 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 239 { 240 if (hb1 > hb2) 241 swap(hb1, hb2); 242 243 spin_lock(&hb1->lock); 244 if (hb1 != hb2) 245 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING); 246 } 247 248 static inline void 249 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 250 { 251 spin_unlock(&hb1->lock); 252 if (hb1 != hb2) 253 spin_unlock(&hb2->lock); 254 } 255 256 /* syscalls */ 257 258 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32 259 val, ktime_t *abs_time, u32 bitset, u32 __user 260 *uaddr2); 261 262 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags, 263 u32 __user *uaddr2, int nr_wake, int nr_requeue, 264 u32 *cmpval, int requeue_pi); 265 266 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, 267 ktime_t *abs_time, u32 bitset); 268 269 /** 270 * struct futex_vector - Auxiliary struct for futex_waitv() 271 * @w: Userspace provided data 272 * @q: Kernel side data 273 * 274 * Struct used to build an array with all data need for futex_waitv() 275 */ 276 struct futex_vector { 277 struct futex_waitv w; 278 struct futex_q q; 279 }; 280 281 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, 282 struct hrtimer_sleeper *to); 283 284 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); 285 286 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags, 287 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op); 288 289 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags); 290 291 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock); 292 293 #endif /* _FUTEX_H */ 294