1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_FUTEX_H 3 #define _LINUX_FUTEX_H 4 5 #include <linux/sched.h> 6 #include <linux/ktime.h> 7 #include <linux/mm_types.h> 8 9 #include <uapi/linux/futex.h> 10 11 struct inode; 12 struct task_struct; 13 14 /* 15 * Futexes are matched on equal values of this key. 16 * The key type depends on whether it's a shared or private mapping. 17 * Don't rearrange members without looking at hash_futex(). 18 * 19 * offset is aligned to a multiple of sizeof(u32) (== 4) by definition. 20 * We use the two low order bits of offset to tell what is the kind of key : 21 * 00 : Private process futex (PTHREAD_PROCESS_PRIVATE) 22 * (no reference on an inode or mm) 23 * 01 : Shared futex (PTHREAD_PROCESS_SHARED) 24 * mapped on a file (reference on the underlying inode) 25 * 10 : Shared futex (PTHREAD_PROCESS_SHARED) 26 * (but private mapping on an mm, and reference taken on it) 27 */ 28 29 #define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */ 30 #define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */ 31 32 union futex_key { 33 struct { 34 u64 i_seq; 35 unsigned long pgoff; 36 unsigned int offset; 37 /* unsigned int node; */ 38 } shared; 39 struct { 40 union { 41 struct mm_struct *mm; 42 u64 __tmp; 43 }; 44 unsigned long address; 45 unsigned int offset; 46 /* unsigned int node; */ 47 } private; 48 struct { 49 u64 ptr; 50 unsigned long word; 51 unsigned int offset; 52 unsigned int node; /* NOT hashed! */ 53 } both; 54 }; 55 56 #define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = 0ULL } } 57 58 #ifdef CONFIG_FUTEX 59 enum { 60 FUTEX_STATE_OK, 61 FUTEX_STATE_EXITING, 62 FUTEX_STATE_DEAD, 63 }; 64 65 static inline void futex_init_task(struct task_struct *tsk) 66 { 67 memset(&tsk->futex, 0, sizeof(tsk->futex)); 68 INIT_LIST_HEAD(&tsk->futex.pi_state_list); 69 tsk->futex.state = FUTEX_STATE_OK; 70 mutex_init(&tsk->futex.exit_mutex); 71 } 72 73 void futex_exit_recursive(struct task_struct *tsk); 74 void futex_exit_release(struct task_struct *tsk); 75 void futex_exec_release(struct task_struct *tsk); 76 77 long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, 78 u32 __user *uaddr2, u32 val2, u32 val3); 79 int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4); 80 81 #ifdef CONFIG_FUTEX_PRIVATE_HASH 82 int futex_hash_allocate_default(void); 83 void futex_hash_free(struct mm_struct *mm); 84 #else /* CONFIG_FUTEX_PRIVATE_HASH */ 85 static inline int futex_hash_allocate_default(void) { return 0; } 86 static inline int futex_hash_free(struct mm_struct *mm) { return 0; } 87 #endif /* !CONFIG_FUTEX_PRIVATE_HASH */ 88 89 #else /* CONFIG_FUTEX */ 90 static inline void futex_init_task(struct task_struct *tsk) { } 91 static inline void futex_exit_recursive(struct task_struct *tsk) { } 92 static inline void futex_exit_release(struct task_struct *tsk) { } 93 static inline void futex_exec_release(struct task_struct *tsk) { } 94 static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, 95 u32 __user *uaddr2, u32 val2, u32 val3) 96 { 97 return -EINVAL; 98 } 99 static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) 100 { 101 return -EINVAL; 102 } 103 static inline int futex_hash_allocate_default(void) { return 0; } 104 static inline int futex_hash_free(struct mm_struct *mm) { return 0; } 105 #endif /* !CONFIG_FUTEX */ 106 107 #ifdef CONFIG_FUTEX_ROBUST_UNLOCK 108 #include <asm/futex_robust.h> 109 110 void futex_reset_cs_ranges(struct futex_mm_data *fd); 111 void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr); 112 113 static inline bool futex_within_robust_unlock(struct pt_regs *regs, 114 struct futex_unlock_cs_range *csr) 115 { 116 unsigned long ip = instruction_pointer(regs); 117 118 return ip >= csr->start_ip && ip < csr->start_ip + csr->len; 119 } 120 121 static inline void futex_fixup_robust_unlock(struct pt_regs *regs) 122 { 123 struct futex_unlock_cs_range *csr; 124 125 /* 126 * Avoid dereferencing current->mm if not returning from interrupt. 127 * current->rseq.event is going to be used subsequently, so bringing the 128 * cache line in is not a big deal. 129 */ 130 if (!current->rseq.event.user_irq) 131 return; 132 133 csr = current->mm->futex.unlock.cs_ranges; 134 135 /* The loop is optimized out for !COMPAT */ 136 for (int r = 0; r < FUTEX_ROBUST_MAX_CS_RANGES; r++, csr++) { 137 if (unlikely(futex_within_robust_unlock(regs, csr))) { 138 __futex_fixup_robust_unlock(regs, csr); 139 return; 140 } 141 } 142 } 143 144 static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx, 145 unsigned long start, unsigned long end, bool sz32) 146 { 147 fd->unlock.cs_ranges[idx].start_ip = start; 148 fd->unlock.cs_ranges[idx].len = end - start; 149 fd->unlock.cs_ranges[idx].pop_size32 = sz32; 150 } 151 #else /* CONFIG_FUTEX_ROBUST_UNLOCK */ 152 static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { } 153 #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ 154 155 156 #if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) 157 void futex_mm_init(struct mm_struct *mm); 158 #else 159 static inline void futex_mm_init(struct mm_struct *mm) { } 160 #endif 161 162 #endif /* _LINUX_FUTEX_H */ 163