1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 ARM Ltd. 4 */ 5 #ifndef __ASM_FUTEX_H 6 #define __ASM_FUTEX_H 7 8 #include <linux/futex.h> 9 #include <linux/uaccess.h> 10 11 #include <asm/errno.h> 12 #include <asm/lsui.h> 13 14 #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */ 15 16 #define LLSC_FUTEX_ATOMIC_OP(op, insn) \ 17 static __always_inline int \ 18 __llsc_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ 19 { \ 20 unsigned int loops = FUTEX_MAX_LOOPS; \ 21 int ret, oldval, newval; \ 22 \ 23 uaccess_enable_privileged(); \ 24 asm volatile("// __llsc_futex_atomic_" #op "\n" \ 25 " prfm pstl1strm, %[uaddr]\n" \ 26 "1: ldxr %w[oldval], %[uaddr]\n" \ 27 insn "\n" \ 28 "2: stlxr %w[ret], %w[newval], %[uaddr]\n" \ 29 " cbz %w[ret], 3f\n" \ 30 " sub %w[loops], %w[loops], %w[ret]\n" \ 31 " cbnz %w[loops], 1b\n" \ 32 " mov %w[ret], %w[err]\n" \ 33 "3:\n" \ 34 " dmb ish\n" \ 35 _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret]) \ 36 _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret]) \ 37 : [ret] "=&r" (ret), [oldval] "=&r" (oldval), \ 38 [uaddr] "+Q" (*uaddr), [newval] "=&r" (newval), \ 39 [loops] "+r" (loops) \ 40 : [oparg] "r" (oparg), [err] "Ir" (-EAGAIN) \ 41 : "memory"); \ 42 uaccess_disable_privileged(); \ 43 \ 44 if (!ret) \ 45 *oval = oldval; \ 46 \ 47 return ret; \ 48 } 49 50 LLSC_FUTEX_ATOMIC_OP(add, "add %w[newval], %w[oldval], %w[oparg]") 51 LLSC_FUTEX_ATOMIC_OP(or, "orr %w[newval], %w[oldval], %w[oparg]") 52 LLSC_FUTEX_ATOMIC_OP(and, "and %w[newval], %w[oldval], %w[oparg]") 53 LLSC_FUTEX_ATOMIC_OP(eor, "eor %w[newval], %w[oldval], %w[oparg]") 54 LLSC_FUTEX_ATOMIC_OP(set, "mov %w[newval], %w[oparg]") 55 56 static __always_inline int 57 __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) 58 { 59 int ret = 0; 60 unsigned int loops = FUTEX_MAX_LOOPS; 61 u32 val, tmp; 62 63 uaccess_enable_privileged(); 64 asm volatile("//__llsc_futex_cmpxchg\n" 65 " prfm pstl1strm, %[uaddr]\n" 66 "1: ldxr %w[curval], %[uaddr]\n" 67 " eor %w[tmp], %w[curval], %w[oldval]\n" 68 " cbnz %w[tmp], 4f\n" 69 "2: stlxr %w[tmp], %w[newval], %[uaddr]\n" 70 " cbz %w[tmp], 3f\n" 71 " sub %w[loops], %w[loops], %w[tmp]\n" 72 " cbnz %w[loops], 1b\n" 73 " mov %w[ret], %w[err]\n" 74 "3:\n" 75 " dmb ish\n" 76 "4:\n" 77 _ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w[ret]) 78 _ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w[ret]) 79 : [ret] "+r" (ret), [curval] "=&r" (val), 80 [uaddr] "+Q" (*uaddr), [tmp] "=&r" (tmp), 81 [loops] "+r" (loops) 82 : [oldval] "r" (oldval), [newval] "r" (newval), 83 [err] "Ir" (-EAGAIN) 84 : "memory"); 85 uaccess_disable_privileged(); 86 87 if (!ret) 88 *oval = val; 89 90 return ret; 91 } 92 93 #ifdef CONFIG_ARM64_LSUI 94 95 /* 96 * Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as 97 * PAN toggling is not required. 98 */ 99 100 #define LSUI_FUTEX_ATOMIC_OP(op, asm_op) \ 101 static __always_inline int \ 102 __lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ 103 { \ 104 int ret = 0; \ 105 int oldval; \ 106 \ 107 uaccess_ttbr0_enable(); \ 108 \ 109 asm volatile("// __lsui_futex_atomic_" #op "\n" \ 110 __LSUI_PREAMBLE \ 111 "1: " #asm_op "al %w[oparg], %w[oldval], %[uaddr]\n" \ 112 "2:\n" \ 113 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) \ 114 : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), \ 115 [oldval] "=r" (oldval) \ 116 : [oparg] "r" (oparg) \ 117 : "memory"); \ 118 \ 119 uaccess_ttbr0_disable(); \ 120 \ 121 if (!ret) \ 122 *oval = oldval; \ 123 return ret; \ 124 } 125 126 LSUI_FUTEX_ATOMIC_OP(add, ldtadd) 127 LSUI_FUTEX_ATOMIC_OP(or, ldtset) 128 LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr) 129 LSUI_FUTEX_ATOMIC_OP(set, swpt) 130 131 static __always_inline int 132 __lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) 133 { 134 int ret = 0; 135 136 uaccess_ttbr0_enable(); 137 138 asm volatile("// __lsui_cmpxchg64\n" 139 __LSUI_PREAMBLE 140 "1: casalt %[oldval], %[newval], %[uaddr]\n" 141 "2:\n" 142 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) 143 : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), 144 [oldval] "+r" (*oldval) 145 : [newval] "r" (newval) 146 : "memory"); 147 148 uaccess_ttbr0_disable(); 149 150 return ret; 151 } 152 153 static __always_inline int 154 __lsui_cmpxchg32(u32 __user *uaddr, u32 *oldval, u32 newval) 155 { 156 u64 __user *uaddr64; 157 bool futex_pos, other_pos; 158 union { 159 u32 futex[2]; 160 u64 raw; 161 } orig64, oval64, nval64; 162 163 uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64)); 164 futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64)); 165 other_pos = !futex_pos; 166 167 orig64.futex[futex_pos] = *oldval; 168 if (get_user(orig64.futex[other_pos], (u32 __user *)uaddr64 + other_pos)) 169 return -EFAULT; 170 171 nval64 = oval64 = orig64; 172 nval64.futex[futex_pos] = newval; 173 174 if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw)) 175 return -EFAULT; 176 177 *oldval = oval64.futex[futex_pos]; 178 return oval64.raw == orig64.raw ? 0 : -EAGAIN; 179 } 180 181 static __always_inline int 182 __lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) 183 { 184 /* 185 * Undo the bitwise negation applied to the oparg passed from 186 * arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN. 187 */ 188 return __lsui_futex_atomic_andnot(~oparg, uaddr, oval); 189 } 190 191 static __always_inline int 192 __lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) 193 { 194 u32 oldval, newval; 195 int ret, i; 196 197 if (get_user(oldval, uaddr)) 198 return -EFAULT; 199 200 /* 201 * there are no ldteor/stteor instructions... 202 */ 203 for (i = 0; i < FUTEX_MAX_LOOPS; i++) { 204 newval = oldval ^ oparg; 205 206 ret = __lsui_cmpxchg32(uaddr, &oldval, newval); 207 if (ret != -EAGAIN) 208 break; 209 } 210 211 *oval = oldval; 212 return ret; 213 } 214 215 static __always_inline int 216 __lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) 217 { 218 u32 curval = oldval; 219 int ret; 220 221 ret = __lsui_cmpxchg32(uaddr, &curval, newval); 222 if (ret == -EAGAIN && curval != oldval) 223 ret = 0; 224 225 *oval = curval; 226 return ret; 227 } 228 #endif /* CONFIG_ARM64_LSUI */ 229 230 231 #define FUTEX_ATOMIC_OP(op) \ 232 static __always_inline int \ 233 __futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ 234 { \ 235 return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \ 236 } 237 238 FUTEX_ATOMIC_OP(add) 239 FUTEX_ATOMIC_OP(or) 240 FUTEX_ATOMIC_OP(and) 241 FUTEX_ATOMIC_OP(eor) 242 FUTEX_ATOMIC_OP(set) 243 244 static __always_inline int 245 __futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) 246 { 247 return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval); 248 } 249 250 static inline int 251 arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr) 252 { 253 int ret; 254 u32 __user *uaddr; 255 256 if (!access_ok(_uaddr, sizeof(u32))) 257 return -EFAULT; 258 259 uaddr = __uaccess_mask_ptr(_uaddr); 260 261 switch (op) { 262 case FUTEX_OP_SET: 263 ret = __futex_atomic_set(oparg, uaddr, oval); 264 break; 265 case FUTEX_OP_ADD: 266 ret = __futex_atomic_add(oparg, uaddr, oval); 267 break; 268 case FUTEX_OP_OR: 269 ret = __futex_atomic_or(oparg, uaddr, oval); 270 break; 271 case FUTEX_OP_ANDN: 272 ret = __futex_atomic_and(~oparg, uaddr, oval); 273 break; 274 case FUTEX_OP_XOR: 275 ret = __futex_atomic_eor(oparg, uaddr, oval); 276 break; 277 default: 278 ret = -ENOSYS; 279 } 280 281 return ret; 282 } 283 284 static inline int 285 futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr, 286 u32 oldval, u32 newval) 287 { 288 u32 __user *uaddr; 289 290 if (!access_ok(_uaddr, sizeof(u32))) 291 return -EFAULT; 292 293 uaddr = __uaccess_mask_ptr(_uaddr); 294 295 return __futex_cmpxchg(uaddr, oldval, newval, uval); 296 } 297 298 #endif /* __ASM_FUTEX_H */ 299