1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _FUTEX_H 3 #define _FUTEX_H 4 5 #include <linux/futex.h> 6 #include <linux/rtmutex.h> 7 #include <linux/sched/wake_q.h> 8 #include <linux/compat.h> 9 10 #ifdef CONFIG_PREEMPT_RT 11 #include <linux/rcuwait.h> 12 #endif 13 14 #include <asm/futex.h> 15 16 /* 17 * Futex flags used to encode options to functions and preserve them across 18 * restarts. 19 */ 20 #define FLAGS_SIZE_8 0x0000 21 #define FLAGS_SIZE_16 0x0001 22 #define FLAGS_SIZE_32 0x0002 23 #define FLAGS_SIZE_64 0x0003 24 25 #define FLAGS_SIZE_MASK 0x0003 26 27 #ifdef CONFIG_MMU 28 # define FLAGS_SHARED 0x0010 29 #else 30 /* 31 * NOMMU does not have per process address space. Let the compiler optimize 32 * code away. 33 */ 34 # define FLAGS_SHARED 0x0000 35 #endif 36 #define FLAGS_CLOCKRT 0x0020 37 #define FLAGS_HAS_TIMEOUT 0x0040 38 #define FLAGS_NUMA 0x0080 39 #define FLAGS_STRICT 0x0100 40 41 /* FUTEX_ to FLAGS_ */ 42 static inline unsigned int futex_to_flags(unsigned int op) 43 { 44 unsigned int flags = FLAGS_SIZE_32; 45 46 if (!(op & FUTEX_PRIVATE_FLAG)) 47 flags |= FLAGS_SHARED; 48 49 if (op & FUTEX_CLOCK_REALTIME) 50 flags |= FLAGS_CLOCKRT; 51 52 return flags; 53 } 54 55 /* FUTEX2_ to FLAGS_ */ 56 static inline unsigned int futex2_to_flags(unsigned int flags2) 57 { 58 unsigned int flags = flags2 & FUTEX2_SIZE_MASK; 59 60 if (!(flags2 & FUTEX2_PRIVATE)) 61 flags |= FLAGS_SHARED; 62 63 if (flags2 & FUTEX2_NUMA) 64 flags |= FLAGS_NUMA; 65 66 return flags; 67 } 68 69 static inline unsigned int futex_size(unsigned int flags) 70 { 71 return 1 << (flags & FLAGS_SIZE_MASK); 72 } 73 74 static inline bool futex_flags_valid(unsigned int flags) 75 { 76 /* Only 64bit futexes for 64bit code */ 77 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) { 78 if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64) 79 return false; 80 } 81 82 /* Only 32bit futexes are implemented -- for now */ 83 if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32) 84 return false; 85 86 return true; 87 } 88 89 static inline bool futex_validate_input(unsigned int flags, u64 val) 90 { 91 int bits = 8 * futex_size(flags); 92 93 if (bits < 64 && (val >> bits)) 94 return false; 95 96 return true; 97 } 98 99 #ifdef CONFIG_FAIL_FUTEX 100 extern bool should_fail_futex(bool fshared); 101 #else 102 static inline bool should_fail_futex(bool fshared) 103 { 104 return false; 105 } 106 #endif 107 108 /* 109 * Hash buckets are shared by all the futex_keys that hash to the same 110 * location. Each key may have multiple futex_q structures, one for each task 111 * waiting on a futex. 112 */ 113 struct futex_hash_bucket { 114 atomic_t waiters; 115 spinlock_t lock; 116 struct plist_head chain; 117 } ____cacheline_aligned_in_smp; 118 119 /* 120 * Priority Inheritance state: 121 */ 122 struct futex_pi_state { 123 /* 124 * list of 'owned' pi_state instances - these have to be 125 * cleaned up in do_exit() if the task exits prematurely: 126 */ 127 struct list_head list; 128 129 /* 130 * The PI object: 131 */ 132 struct rt_mutex_base pi_mutex; 133 134 struct task_struct *owner; 135 refcount_t refcount; 136 137 union futex_key key; 138 } __randomize_layout; 139 140 /** 141 * struct futex_q - The hashed futex queue entry, one per waiting task 142 * @list: priority-sorted list of tasks waiting on this futex 143 * @task: the task waiting on the futex 144 * @lock_ptr: the hash bucket lock 145 * @key: the key the futex is hashed on 146 * @pi_state: optional priority inheritance state 147 * @rt_waiter: rt_waiter storage for use with requeue_pi 148 * @requeue_pi_key: the requeue_pi target futex key 149 * @bitset: bitset for the optional bitmasked wakeup 150 * @requeue_state: State field for futex_requeue_pi() 151 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) 152 * 153 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so 154 * we can wake only the relevant ones (hashed queues may be shared). 155 * 156 * A futex_q has a woken state, just like tasks have TASK_RUNNING. 157 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. 158 * The order of wakeup is always to make the first condition true, then 159 * the second. 160 * 161 * PI futexes are typically woken before they are removed from the hash list via 162 * the rt_mutex code. See futex_unqueue_pi(). 163 */ 164 struct futex_q { 165 struct plist_node list; 166 167 struct task_struct *task; 168 spinlock_t *lock_ptr; 169 union futex_key key; 170 struct futex_pi_state *pi_state; 171 struct rt_mutex_waiter *rt_waiter; 172 union futex_key *requeue_pi_key; 173 u32 bitset; 174 atomic_t requeue_state; 175 #ifdef CONFIG_PREEMPT_RT 176 struct rcuwait requeue_wait; 177 #endif 178 } __randomize_layout; 179 180 extern const struct futex_q futex_q_init; 181 182 enum futex_access { 183 FUTEX_READ, 184 FUTEX_WRITE 185 }; 186 187 extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, 188 enum futex_access rw); 189 190 extern struct hrtimer_sleeper * 191 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 192 int flags, u64 range_ns); 193 194 extern struct futex_hash_bucket *futex_hash(union futex_key *key); 195 196 /** 197 * futex_match - Check whether two futex keys are equal 198 * @key1: Pointer to key1 199 * @key2: Pointer to key2 200 * 201 * Return 1 if two futex_keys are equal, 0 otherwise. 202 */ 203 static inline int futex_match(union futex_key *key1, union futex_key *key2) 204 { 205 return (key1 && key2 206 && key1->both.word == key2->both.word 207 && key1->both.ptr == key2->both.ptr 208 && key1->both.offset == key2->both.offset); 209 } 210 211 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 212 struct futex_q *q, struct futex_hash_bucket **hb); 213 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, 214 struct hrtimer_sleeper *timeout); 215 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q); 216 217 extern int fault_in_user_writeable(u32 __user *uaddr); 218 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval); 219 extern int futex_get_value_locked(u32 *dest, u32 __user *from); 220 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key); 221 222 extern void __futex_unqueue(struct futex_q *q); 223 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb); 224 extern int futex_unqueue(struct futex_q *q); 225 226 /** 227 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket 228 * @q: The futex_q to enqueue 229 * @hb: The destination hash bucket 230 * 231 * The hb->lock must be held by the caller, and is released here. A call to 232 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The 233 * exceptions involve the PI related operations, which may use futex_unqueue_pi() 234 * or nothing if the unqueue is done as part of the wake process and the unqueue 235 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for 236 * an example). 237 */ 238 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb) 239 __releases(&hb->lock) 240 { 241 __futex_queue(q, hb); 242 spin_unlock(&hb->lock); 243 } 244 245 extern void futex_unqueue_pi(struct futex_q *q); 246 247 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting); 248 249 /* 250 * Reflects a new waiter being added to the waitqueue. 251 */ 252 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb) 253 { 254 #ifdef CONFIG_SMP 255 atomic_inc(&hb->waiters); 256 /* 257 * Full barrier (A), see the ordering comment above. 258 */ 259 smp_mb__after_atomic(); 260 #endif 261 } 262 263 /* 264 * Reflects a waiter being removed from the waitqueue by wakeup 265 * paths. 266 */ 267 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb) 268 { 269 #ifdef CONFIG_SMP 270 atomic_dec(&hb->waiters); 271 #endif 272 } 273 274 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb) 275 { 276 #ifdef CONFIG_SMP 277 /* 278 * Full barrier (B), see the ordering comment above. 279 */ 280 smp_mb(); 281 return atomic_read(&hb->waiters); 282 #else 283 return 1; 284 #endif 285 } 286 287 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q); 288 extern void futex_q_unlock(struct futex_hash_bucket *hb); 289 290 291 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb, 292 union futex_key *key, 293 struct futex_pi_state **ps, 294 struct task_struct *task, 295 struct task_struct **exiting, 296 int set_waiters); 297 298 extern int refill_pi_state_cache(void); 299 extern void get_pi_state(struct futex_pi_state *pi_state); 300 extern void put_pi_state(struct futex_pi_state *pi_state); 301 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked); 302 303 /* 304 * Express the locking dependencies for lockdep: 305 */ 306 static inline void 307 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 308 { 309 if (hb1 > hb2) 310 swap(hb1, hb2); 311 312 spin_lock(&hb1->lock); 313 if (hb1 != hb2) 314 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING); 315 } 316 317 static inline void 318 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 319 { 320 spin_unlock(&hb1->lock); 321 if (hb1 != hb2) 322 spin_unlock(&hb2->lock); 323 } 324 325 /* syscalls */ 326 327 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32 328 val, ktime_t *abs_time, u32 bitset, u32 __user 329 *uaddr2); 330 331 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1, 332 u32 __user *uaddr2, unsigned int flags2, 333 int nr_wake, int nr_requeue, 334 u32 *cmpval, int requeue_pi); 335 336 extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, 337 struct hrtimer_sleeper *to, u32 bitset); 338 339 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, 340 ktime_t *abs_time, u32 bitset); 341 342 /** 343 * struct futex_vector - Auxiliary struct for futex_waitv() 344 * @w: Userspace provided data 345 * @q: Kernel side data 346 * 347 * Struct used to build an array with all data need for futex_waitv() 348 */ 349 struct futex_vector { 350 struct futex_waitv w; 351 struct futex_q q; 352 }; 353 354 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, 355 struct hrtimer_sleeper *to); 356 357 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); 358 359 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags, 360 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op); 361 362 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags); 363 364 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock); 365 366 #endif /* _FUTEX_H */ 367