1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_FUTEX_H
3 #define _LINUX_FUTEX_H
4
5 #include <linux/sched.h>
6 #include <linux/ktime.h>
7 #include <linux/mm_types.h>
8
9 #include <uapi/linux/futex.h>
10
11 struct inode;
12 struct task_struct;
13
14 /*
15 * Futexes are matched on equal values of this key.
16 * The key type depends on whether it's a shared or private mapping.
17 * Don't rearrange members without looking at hash_futex().
18 *
19 * offset is aligned to a multiple of sizeof(u32) (== 4) by definition.
20 * We use the two low order bits of offset to tell what is the kind of key :
21 * 00 : Private process futex (PTHREAD_PROCESS_PRIVATE)
22 * (no reference on an inode or mm)
23 * 01 : Shared futex (PTHREAD_PROCESS_SHARED)
24 * mapped on a file (reference on the underlying inode)
25 * 10 : Shared futex (PTHREAD_PROCESS_SHARED)
26 * (but private mapping on an mm)
27 */
28
29 #define FUT_OFF_INODE 1 /* We set bit 0 if shared key identifies an inode */
30 #define FUT_OFF_MMSHARED 2 /* We set bit 1 if shared key identifies an mm */
31
32 union futex_key {
33 /* For FUT_OFF_INODE */
34 struct {
35 u64 i_seq;
36 unsigned long pgoff;
37 unsigned int offset;
38 /* unsigned int node; */
39 } shared;
40
41 /* For FUT_OFF_MMSHARED or private */
42 struct {
43 union {
44 struct mm_struct *mm; /* for private */
45 u64 mm_seq; /* for FUT_OFF_MMSHARED */
46 };
47 unsigned long address;
48 unsigned int offset;
49 /* unsigned int node; */
50 } private;
51
52 struct {
53 u64 ptr;
54 unsigned long word;
55 unsigned int offset;
56 unsigned int node; /* NOT hashed! */
57 } both;
58 };
59
60 #define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = 0ULL } }
61
62 #ifdef CONFIG_FUTEX
63 enum {
64 FUTEX_STATE_OK,
65 FUTEX_STATE_EXITING,
66 FUTEX_STATE_DEAD,
67 };
68
futex_init_task(struct task_struct * tsk)69 static inline void futex_init_task(struct task_struct *tsk)
70 {
71 memset(&tsk->futex, 0, sizeof(tsk->futex));
72 INIT_LIST_HEAD(&tsk->futex.pi_state_list);
73 tsk->futex.state = FUTEX_STATE_OK;
74 mutex_init(&tsk->futex.exit_mutex);
75 }
76
77 void futex_exit_recursive(struct task_struct *tsk);
78 void futex_exit_exec_release(struct task_struct *tsk);
79 void futex_exec_done(struct task_struct *tsk);
80
81 long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
82 u32 __user *uaddr2, u32 val2, u32 val3);
83 int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4);
84
85 #ifdef CONFIG_FUTEX_PRIVATE_HASH
86 int futex_hash_allocate_default(void);
87 void futex_hash_free(struct mm_struct *mm);
88 #else /* CONFIG_FUTEX_PRIVATE_HASH */
futex_hash_allocate_default(void)89 static inline int futex_hash_allocate_default(void) { return 0; }
futex_hash_free(struct mm_struct * mm)90 static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
91 #endif /* !CONFIG_FUTEX_PRIVATE_HASH */
92
93 #else /* CONFIG_FUTEX */
futex_init_task(struct task_struct * tsk)94 static inline void futex_init_task(struct task_struct *tsk) { }
futex_exit_recursive(struct task_struct * tsk)95 static inline void futex_exit_recursive(struct task_struct *tsk) { }
futex_exit_exec_release(struct task_struct * tsk)96 static inline void futex_exit_exec_release(struct task_struct *tsk) { }
futex_exec_done(struct task_struct * tsk)97 static inline void futex_exec_done(struct task_struct *tsk) { }
do_futex(u32 __user * uaddr,int op,u32 val,ktime_t * timeout,u32 __user * uaddr2,u32 val2,u32 val3)98 static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
99 u32 __user *uaddr2, u32 val2, u32 val3)
100 {
101 return -EINVAL;
102 }
futex_hash_prctl(unsigned long arg2,unsigned long arg3,unsigned long arg4)103 static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
104 {
105 return -EINVAL;
106 }
futex_hash_allocate_default(void)107 static inline int futex_hash_allocate_default(void) { return 0; }
futex_hash_free(struct mm_struct * mm)108 static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
109 #endif /* !CONFIG_FUTEX */
110
111 #ifdef CONFIG_FUTEX_ROBUST_UNLOCK
112 #include <asm/futex_robust.h>
113
114 void futex_reset_cs_ranges(struct futex_mm_data *fd);
115 void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr);
116
futex_within_robust_unlock(struct pt_regs * regs,struct futex_unlock_cs_range * csr)117 static inline bool futex_within_robust_unlock(struct pt_regs *regs,
118 struct futex_unlock_cs_range *csr)
119 {
120 unsigned long ip = instruction_pointer(regs);
121
122 return ip >= csr->start_ip && ip < csr->start_ip + csr->len;
123 }
124
futex_fixup_robust_unlock(struct pt_regs * regs)125 static inline void futex_fixup_robust_unlock(struct pt_regs *regs)
126 {
127 struct futex_unlock_cs_range *csr;
128
129 /*
130 * Avoid dereferencing current->mm if not returning from interrupt.
131 * current->rseq.event is going to be used subsequently, so bringing the
132 * cache line in is not a big deal.
133 */
134 if (!current->rseq.event.user_irq)
135 return;
136
137 csr = current->mm->futex.unlock.cs_ranges;
138
139 /* The loop is optimized out for !COMPAT */
140 for (int r = 0; r < FUTEX_ROBUST_MAX_CS_RANGES; r++, csr++) {
141 if (unlikely(futex_within_robust_unlock(regs, csr))) {
142 __futex_fixup_robust_unlock(regs, csr);
143 return;
144 }
145 }
146 }
147
futex_set_vdso_cs_range(struct futex_mm_data * fd,unsigned int idx,unsigned long start,unsigned long end,bool sz32)148 static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx,
149 unsigned long start, unsigned long end, bool sz32)
150 {
151 fd->unlock.cs_ranges[idx].start_ip = start;
152 fd->unlock.cs_ranges[idx].len = end - start;
153 fd->unlock.cs_ranges[idx].pop_size32 = sz32;
154 }
155 #else /* CONFIG_FUTEX_ROBUST_UNLOCK */
futex_fixup_robust_unlock(struct pt_regs * regs)156 static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { }
157 #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */
158
159 #ifdef CONFIG_FUTEX
160 void futex_mm_init(struct mm_struct *mm);
161 #else
futex_mm_init(struct mm_struct * mm)162 static inline void futex_mm_init(struct mm_struct *mm) { }
163 #endif /* !CONFIG_FUTEX */
164
165 #endif /* _LINUX_FUTEX_H */
166