1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _FUTEX_H 3 #define _FUTEX_H 4 5 #include <linux/futex.h> 6 #include <linux/rtmutex.h> 7 #include <linux/sched/wake_q.h> 8 #include <linux/compat.h> 9 10 #ifdef CONFIG_PREEMPT_RT 11 #include <linux/rcuwait.h> 12 #endif 13 14 #include <asm/futex.h> 15 16 /* 17 * Futex flags used to encode options to functions and preserve them across 18 * restarts. 19 */ 20 #define FLAGS_SIZE_8 0x0000 21 #define FLAGS_SIZE_16 0x0001 22 #define FLAGS_SIZE_32 0x0002 23 #define FLAGS_SIZE_64 0x0003 24 25 #define FLAGS_SIZE_MASK 0x0003 26 27 #ifdef CONFIG_MMU 28 # define FLAGS_SHARED 0x0010 29 #else 30 /* 31 * NOMMU does not have per process address space. Let the compiler optimize 32 * code away. 33 */ 34 # define FLAGS_SHARED 0x0000 35 #endif 36 #define FLAGS_CLOCKRT 0x0020 37 #define FLAGS_HAS_TIMEOUT 0x0040 38 #define FLAGS_NUMA 0x0080 39 #define FLAGS_STRICT 0x0100 40 41 /* FUTEX_ to FLAGS_ */ 42 static inline unsigned int futex_to_flags(unsigned int op) 43 { 44 unsigned int flags = FLAGS_SIZE_32; 45 46 if (!(op & FUTEX_PRIVATE_FLAG)) 47 flags |= FLAGS_SHARED; 48 49 if (op & FUTEX_CLOCK_REALTIME) 50 flags |= FLAGS_CLOCKRT; 51 52 return flags; 53 } 54 55 #define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE) 56 57 /* FUTEX2_ to FLAGS_ */ 58 static inline unsigned int futex2_to_flags(unsigned int flags2) 59 { 60 unsigned int flags = flags2 & FUTEX2_SIZE_MASK; 61 62 if (!(flags2 & FUTEX2_PRIVATE)) 63 flags |= FLAGS_SHARED; 64 65 if (flags2 & FUTEX2_NUMA) 66 flags |= FLAGS_NUMA; 67 68 return flags; 69 } 70 71 static inline unsigned int futex_size(unsigned int flags) 72 { 73 return 1 << (flags & FLAGS_SIZE_MASK); 74 } 75 76 static inline bool futex_flags_valid(unsigned int flags) 77 { 78 /* Only 64bit futexes for 64bit code */ 79 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) { 80 if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64) 81 return false; 82 } 83 84 /* Only 32bit futexes are implemented -- for now */ 85 if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32) 86 return false; 87 88 return true; 89 } 90 91 static inline bool futex_validate_input(unsigned int flags, u64 val) 92 { 93 int bits = 8 * futex_size(flags); 94 95 if (bits < 64 && (val >> bits)) 96 return false; 97 98 return true; 99 } 100 101 #ifdef CONFIG_FAIL_FUTEX 102 extern bool should_fail_futex(bool fshared); 103 #else 104 static inline bool should_fail_futex(bool fshared) 105 { 106 return false; 107 } 108 #endif 109 110 /* 111 * Hash buckets are shared by all the futex_keys that hash to the same 112 * location. Each key may have multiple futex_q structures, one for each task 113 * waiting on a futex. 114 */ 115 struct futex_hash_bucket { 116 atomic_t waiters; 117 spinlock_t lock; 118 struct plist_head chain; 119 } ____cacheline_aligned_in_smp; 120 121 /* 122 * Priority Inheritance state: 123 */ 124 struct futex_pi_state { 125 /* 126 * list of 'owned' pi_state instances - these have to be 127 * cleaned up in do_exit() if the task exits prematurely: 128 */ 129 struct list_head list; 130 131 /* 132 * The PI object: 133 */ 134 struct rt_mutex_base pi_mutex; 135 136 struct task_struct *owner; 137 refcount_t refcount; 138 139 union futex_key key; 140 } __randomize_layout; 141 142 struct futex_q; 143 typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q); 144 145 /** 146 * struct futex_q - The hashed futex queue entry, one per waiting task 147 * @list: priority-sorted list of tasks waiting on this futex 148 * @task: the task waiting on the futex 149 * @lock_ptr: the hash bucket lock 150 * @wake: the wake handler for this queue 151 * @wake_data: data associated with the wake handler 152 * @key: the key the futex is hashed on 153 * @pi_state: optional priority inheritance state 154 * @rt_waiter: rt_waiter storage for use with requeue_pi 155 * @requeue_pi_key: the requeue_pi target futex key 156 * @bitset: bitset for the optional bitmasked wakeup 157 * @requeue_state: State field for futex_requeue_pi() 158 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) 159 * 160 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so 161 * we can wake only the relevant ones (hashed queues may be shared). 162 * 163 * A futex_q has a woken state, just like tasks have TASK_RUNNING. 164 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. 165 * The order of wakeup is always to make the first condition true, then 166 * the second. 167 * 168 * PI futexes are typically woken before they are removed from the hash list via 169 * the rt_mutex code. See futex_unqueue_pi(). 170 */ 171 struct futex_q { 172 struct plist_node list; 173 174 struct task_struct *task; 175 spinlock_t *lock_ptr; 176 futex_wake_fn *wake; 177 void *wake_data; 178 union futex_key key; 179 struct futex_pi_state *pi_state; 180 struct rt_mutex_waiter *rt_waiter; 181 union futex_key *requeue_pi_key; 182 u32 bitset; 183 atomic_t requeue_state; 184 #ifdef CONFIG_PREEMPT_RT 185 struct rcuwait requeue_wait; 186 #endif 187 } __randomize_layout; 188 189 extern const struct futex_q futex_q_init; 190 191 enum futex_access { 192 FUTEX_READ, 193 FUTEX_WRITE 194 }; 195 196 extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, 197 enum futex_access rw); 198 199 extern struct hrtimer_sleeper * 200 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, 201 int flags, u64 range_ns); 202 203 extern struct futex_hash_bucket *futex_hash(union futex_key *key); 204 205 /** 206 * futex_match - Check whether two futex keys are equal 207 * @key1: Pointer to key1 208 * @key2: Pointer to key2 209 * 210 * Return 1 if two futex_keys are equal, 0 otherwise. 211 */ 212 static inline int futex_match(union futex_key *key1, union futex_key *key2) 213 { 214 return (key1 && key2 215 && key1->both.word == key2->both.word 216 && key1->both.ptr == key2->both.ptr 217 && key1->both.offset == key2->both.offset); 218 } 219 220 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, 221 struct futex_q *q, struct futex_hash_bucket **hb); 222 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, 223 struct hrtimer_sleeper *timeout); 224 extern bool __futex_wake_mark(struct futex_q *q); 225 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q); 226 227 extern int fault_in_user_writeable(u32 __user *uaddr); 228 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval); 229 extern int futex_get_value_locked(u32 *dest, u32 __user *from); 230 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key); 231 232 extern void __futex_unqueue(struct futex_q *q); 233 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb); 234 extern int futex_unqueue(struct futex_q *q); 235 236 /** 237 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket 238 * @q: The futex_q to enqueue 239 * @hb: The destination hash bucket 240 * 241 * The hb->lock must be held by the caller, and is released here. A call to 242 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The 243 * exceptions involve the PI related operations, which may use futex_unqueue_pi() 244 * or nothing if the unqueue is done as part of the wake process and the unqueue 245 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for 246 * an example). 247 */ 248 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb) 249 __releases(&hb->lock) 250 { 251 __futex_queue(q, hb); 252 spin_unlock(&hb->lock); 253 } 254 255 extern void futex_unqueue_pi(struct futex_q *q); 256 257 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting); 258 259 /* 260 * Reflects a new waiter being added to the waitqueue. 261 */ 262 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb) 263 { 264 #ifdef CONFIG_SMP 265 atomic_inc(&hb->waiters); 266 /* 267 * Full barrier (A), see the ordering comment above. 268 */ 269 smp_mb__after_atomic(); 270 #endif 271 } 272 273 /* 274 * Reflects a waiter being removed from the waitqueue by wakeup 275 * paths. 276 */ 277 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb) 278 { 279 #ifdef CONFIG_SMP 280 atomic_dec(&hb->waiters); 281 #endif 282 } 283 284 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb) 285 { 286 #ifdef CONFIG_SMP 287 /* 288 * Full barrier (B), see the ordering comment above. 289 */ 290 smp_mb(); 291 return atomic_read(&hb->waiters); 292 #else 293 return 1; 294 #endif 295 } 296 297 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q); 298 extern void futex_q_unlock(struct futex_hash_bucket *hb); 299 300 301 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb, 302 union futex_key *key, 303 struct futex_pi_state **ps, 304 struct task_struct *task, 305 struct task_struct **exiting, 306 int set_waiters); 307 308 extern int refill_pi_state_cache(void); 309 extern void get_pi_state(struct futex_pi_state *pi_state); 310 extern void put_pi_state(struct futex_pi_state *pi_state); 311 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked); 312 313 /* 314 * Express the locking dependencies for lockdep: 315 */ 316 static inline void 317 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 318 { 319 if (hb1 > hb2) 320 swap(hb1, hb2); 321 322 spin_lock(&hb1->lock); 323 if (hb1 != hb2) 324 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING); 325 } 326 327 static inline void 328 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) 329 { 330 spin_unlock(&hb1->lock); 331 if (hb1 != hb2) 332 spin_unlock(&hb2->lock); 333 } 334 335 /* syscalls */ 336 337 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32 338 val, ktime_t *abs_time, u32 bitset, u32 __user 339 *uaddr2); 340 341 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1, 342 u32 __user *uaddr2, unsigned int flags2, 343 int nr_wake, int nr_requeue, 344 u32 *cmpval, int requeue_pi); 345 346 extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, 347 struct hrtimer_sleeper *to, u32 bitset); 348 349 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, 350 ktime_t *abs_time, u32 bitset); 351 352 /** 353 * struct futex_vector - Auxiliary struct for futex_waitv() 354 * @w: Userspace provided data 355 * @q: Kernel side data 356 * 357 * Struct used to build an array with all data need for futex_waitv() 358 */ 359 struct futex_vector { 360 struct futex_waitv w; 361 struct futex_q q; 362 }; 363 364 extern int futex_parse_waitv(struct futex_vector *futexv, 365 struct futex_waitv __user *uwaitv, 366 unsigned int nr_futexes, futex_wake_fn *wake, 367 void *wake_data); 368 369 extern int futex_wait_multiple_setup(struct futex_vector *vs, int count, 370 int *woken); 371 372 extern int futex_unqueue_multiple(struct futex_vector *v, int count); 373 374 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, 375 struct hrtimer_sleeper *to); 376 377 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); 378 379 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags, 380 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op); 381 382 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags); 383 384 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock); 385 386 #endif /* _FUTEX_H */ 387