1 /* 2 * Based on arch/arm/include/asm/atomic.h 3 * 4 * Copyright (C) 1996 Russell King. 5 * Copyright (C) 2002 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #ifndef __ASM_ATOMIC_LSE_H 22 #define __ASM_ATOMIC_LSE_H 23 24 #ifndef __ARM64_IN_ATOMIC_IMPL 25 #error "please don't include this file directly" 26 #endif 27 28 #define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op) 29 30 static inline void atomic_andnot(int i, atomic_t *v) 31 { 32 register int w0 asm ("w0") = i; 33 register atomic_t *x1 asm ("x1") = v; 34 35 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot), 36 " stclr %w[i], %[v]\n") 37 : [i] "+r" (w0), [v] "+Q" (v->counter) 38 : "r" (x1) 39 : "x30"); 40 } 41 42 static inline void atomic_or(int i, atomic_t *v) 43 { 44 register int w0 asm ("w0") = i; 45 register atomic_t *x1 asm ("x1") = v; 46 47 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or), 48 " stset %w[i], %[v]\n") 49 : [i] "+r" (w0), [v] "+Q" (v->counter) 50 : "r" (x1) 51 : "x30"); 52 } 53 54 static inline void atomic_xor(int i, atomic_t *v) 55 { 56 register int w0 asm ("w0") = i; 57 register atomic_t *x1 asm ("x1") = v; 58 59 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor), 60 " steor %w[i], %[v]\n") 61 : [i] "+r" (w0), [v] "+Q" (v->counter) 62 : "r" (x1) 63 : "x30"); 64 } 65 66 static inline void atomic_add(int i, atomic_t *v) 67 { 68 register int w0 asm ("w0") = i; 69 register atomic_t *x1 asm ("x1") = v; 70 71 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add), 72 " stadd %w[i], %[v]\n") 73 : [i] "+r" (w0), [v] "+Q" (v->counter) 74 : "r" (x1) 75 : "x30"); 76 } 77 78 static inline int atomic_add_return(int i, atomic_t *v) 79 { 80 register int w0 asm ("w0") = i; 81 register atomic_t *x1 asm ("x1") = v; 82 83 asm volatile(ARM64_LSE_ATOMIC_INSN( 84 /* LL/SC */ 85 " nop\n" 86 __LL_SC_ATOMIC(add_return), 87 /* LSE atomics */ 88 " ldaddal %w[i], w30, %[v]\n" 89 " add %w[i], %w[i], w30") 90 : [i] "+r" (w0), [v] "+Q" (v->counter) 91 : "r" (x1) 92 : "x30", "memory"); 93 94 return w0; 95 } 96 97 static inline void atomic_and(int i, atomic_t *v) 98 { 99 register int w0 asm ("w0") = i; 100 register atomic_t *x1 asm ("x1") = v; 101 102 asm volatile(ARM64_LSE_ATOMIC_INSN( 103 /* LL/SC */ 104 " nop\n" 105 __LL_SC_ATOMIC(and), 106 /* LSE atomics */ 107 " mvn %w[i], %w[i]\n" 108 " stclr %w[i], %[v]") 109 : [i] "+r" (w0), [v] "+Q" (v->counter) 110 : "r" (x1) 111 : "x30"); 112 } 113 114 static inline void atomic_sub(int i, atomic_t *v) 115 { 116 register int w0 asm ("w0") = i; 117 register atomic_t *x1 asm ("x1") = v; 118 119 asm volatile(ARM64_LSE_ATOMIC_INSN( 120 /* LL/SC */ 121 " nop\n" 122 __LL_SC_ATOMIC(sub), 123 /* LSE atomics */ 124 " neg %w[i], %w[i]\n" 125 " stadd %w[i], %[v]") 126 : [i] "+r" (w0), [v] "+Q" (v->counter) 127 : "r" (x1) 128 : "x30"); 129 } 130 131 static inline int atomic_sub_return(int i, atomic_t *v) 132 { 133 register int w0 asm ("w0") = i; 134 register atomic_t *x1 asm ("x1") = v; 135 136 asm volatile(ARM64_LSE_ATOMIC_INSN( 137 /* LL/SC */ 138 " nop\n" 139 __LL_SC_ATOMIC(sub_return) 140 " nop", 141 /* LSE atomics */ 142 " neg %w[i], %w[i]\n" 143 " ldaddal %w[i], w30, %[v]\n" 144 " add %w[i], %w[i], w30") 145 : [i] "+r" (w0), [v] "+Q" (v->counter) 146 : "r" (x1) 147 : "x30", "memory"); 148 149 return w0; 150 } 151 152 #undef __LL_SC_ATOMIC 153 154 #define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) 155 156 static inline void atomic64_andnot(long i, atomic64_t *v) 157 { 158 register long x0 asm ("x0") = i; 159 register atomic64_t *x1 asm ("x1") = v; 160 161 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot), 162 " stclr %[i], %[v]\n") 163 : [i] "+r" (x0), [v] "+Q" (v->counter) 164 : "r" (x1) 165 : "x30"); 166 } 167 168 static inline void atomic64_or(long i, atomic64_t *v) 169 { 170 register long x0 asm ("x0") = i; 171 register atomic64_t *x1 asm ("x1") = v; 172 173 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or), 174 " stset %[i], %[v]\n") 175 : [i] "+r" (x0), [v] "+Q" (v->counter) 176 : "r" (x1) 177 : "x30"); 178 } 179 180 static inline void atomic64_xor(long i, atomic64_t *v) 181 { 182 register long x0 asm ("x0") = i; 183 register atomic64_t *x1 asm ("x1") = v; 184 185 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor), 186 " steor %[i], %[v]\n") 187 : [i] "+r" (x0), [v] "+Q" (v->counter) 188 : "r" (x1) 189 : "x30"); 190 } 191 192 static inline void atomic64_add(long i, atomic64_t *v) 193 { 194 register long x0 asm ("x0") = i; 195 register atomic64_t *x1 asm ("x1") = v; 196 197 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add), 198 " stadd %[i], %[v]\n") 199 : [i] "+r" (x0), [v] "+Q" (v->counter) 200 : "r" (x1) 201 : "x30"); 202 } 203 204 static inline long atomic64_add_return(long i, atomic64_t *v) 205 { 206 register long x0 asm ("x0") = i; 207 register atomic64_t *x1 asm ("x1") = v; 208 209 asm volatile(ARM64_LSE_ATOMIC_INSN( 210 /* LL/SC */ 211 " nop\n" 212 __LL_SC_ATOMIC64(add_return), 213 /* LSE atomics */ 214 " ldaddal %[i], x30, %[v]\n" 215 " add %[i], %[i], x30") 216 : [i] "+r" (x0), [v] "+Q" (v->counter) 217 : "r" (x1) 218 : "x30", "memory"); 219 220 return x0; 221 } 222 223 static inline void atomic64_and(long i, atomic64_t *v) 224 { 225 register long x0 asm ("x0") = i; 226 register atomic64_t *x1 asm ("x1") = v; 227 228 asm volatile(ARM64_LSE_ATOMIC_INSN( 229 /* LL/SC */ 230 " nop\n" 231 __LL_SC_ATOMIC64(and), 232 /* LSE atomics */ 233 " mvn %[i], %[i]\n" 234 " stclr %[i], %[v]") 235 : [i] "+r" (x0), [v] "+Q" (v->counter) 236 : "r" (x1) 237 : "x30"); 238 } 239 240 static inline void atomic64_sub(long i, atomic64_t *v) 241 { 242 register long x0 asm ("x0") = i; 243 register atomic64_t *x1 asm ("x1") = v; 244 245 asm volatile(ARM64_LSE_ATOMIC_INSN( 246 /* LL/SC */ 247 " nop\n" 248 __LL_SC_ATOMIC64(sub), 249 /* LSE atomics */ 250 " neg %[i], %[i]\n" 251 " stadd %[i], %[v]") 252 : [i] "+r" (x0), [v] "+Q" (v->counter) 253 : "r" (x1) 254 : "x30"); 255 } 256 257 static inline long atomic64_sub_return(long i, atomic64_t *v) 258 { 259 register long x0 asm ("x0") = i; 260 register atomic64_t *x1 asm ("x1") = v; 261 262 asm volatile(ARM64_LSE_ATOMIC_INSN( 263 /* LL/SC */ 264 " nop\n" 265 __LL_SC_ATOMIC64(sub_return) 266 " nop", 267 /* LSE atomics */ 268 " neg %[i], %[i]\n" 269 " ldaddal %[i], x30, %[v]\n" 270 " add %[i], %[i], x30") 271 : [i] "+r" (x0), [v] "+Q" (v->counter) 272 : "r" (x1) 273 : "x30", "memory"); 274 275 return x0; 276 } 277 278 static inline long atomic64_dec_if_positive(atomic64_t *v) 279 { 280 register long x0 asm ("x0") = (long)v; 281 282 asm volatile(ARM64_LSE_ATOMIC_INSN( 283 /* LL/SC */ 284 " nop\n" 285 __LL_SC_ATOMIC64(dec_if_positive) 286 " nop\n" 287 " nop\n" 288 " nop\n" 289 " nop\n" 290 " nop", 291 /* LSE atomics */ 292 "1: ldr x30, %[v]\n" 293 " subs %[ret], x30, #1\n" 294 " b.lt 2f\n" 295 " casal x30, %[ret], %[v]\n" 296 " sub x30, x30, #1\n" 297 " sub x30, x30, %[ret]\n" 298 " cbnz x30, 1b\n" 299 "2:") 300 : [ret] "+&r" (x0), [v] "+Q" (v->counter) 301 : 302 : "x30", "cc", "memory"); 303 304 return x0; 305 } 306 307 #undef __LL_SC_ATOMIC64 308 309 #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op) 310 311 #define __CMPXCHG_CASE(w, sz, name, mb, cl...) \ 312 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ 313 unsigned long old, \ 314 unsigned long new) \ 315 { \ 316 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ 317 register unsigned long x1 asm ("x1") = old; \ 318 register unsigned long x2 asm ("x2") = new; \ 319 \ 320 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 321 /* LL/SC */ \ 322 " nop\n" \ 323 __LL_SC_CMPXCHG(name) \ 324 " nop", \ 325 /* LSE atomics */ \ 326 " mov " #w "30, %" #w "[old]\n" \ 327 " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \ 328 " mov %" #w "[ret], " #w "30") \ 329 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \ 330 : [old] "r" (x1), [new] "r" (x2) \ 331 : "x30" , ##cl); \ 332 \ 333 return x0; \ 334 } 335 336 __CMPXCHG_CASE(w, b, 1, ) 337 __CMPXCHG_CASE(w, h, 2, ) 338 __CMPXCHG_CASE(w, , 4, ) 339 __CMPXCHG_CASE(x, , 8, ) 340 __CMPXCHG_CASE(w, b, mb_1, al, "memory") 341 __CMPXCHG_CASE(w, h, mb_2, al, "memory") 342 __CMPXCHG_CASE(w, , mb_4, al, "memory") 343 __CMPXCHG_CASE(x, , mb_8, al, "memory") 344 345 #undef __LL_SC_CMPXCHG 346 #undef __CMPXCHG_CASE 347 348 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op) 349 350 #define __CMPXCHG_DBL(name, mb, cl...) \ 351 static inline int __cmpxchg_double##name(unsigned long old1, \ 352 unsigned long old2, \ 353 unsigned long new1, \ 354 unsigned long new2, \ 355 volatile void *ptr) \ 356 { \ 357 unsigned long oldval1 = old1; \ 358 unsigned long oldval2 = old2; \ 359 register unsigned long x0 asm ("x0") = old1; \ 360 register unsigned long x1 asm ("x1") = old2; \ 361 register unsigned long x2 asm ("x2") = new1; \ 362 register unsigned long x3 asm ("x3") = new2; \ 363 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ 364 \ 365 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 366 /* LL/SC */ \ 367 " nop\n" \ 368 " nop\n" \ 369 " nop\n" \ 370 __LL_SC_CMPXCHG_DBL(name), \ 371 /* LSE atomics */ \ 372 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ 373 " eor %[old1], %[old1], %[oldval1]\n" \ 374 " eor %[old2], %[old2], %[oldval2]\n" \ 375 " orr %[old1], %[old1], %[old2]") \ 376 : [old1] "+r" (x0), [old2] "+r" (x1), \ 377 [v] "+Q" (*(unsigned long *)ptr) \ 378 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ 379 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ 380 : "x30" , ##cl); \ 381 \ 382 return x0; \ 383 } 384 385 __CMPXCHG_DBL( , ) 386 __CMPXCHG_DBL(_mb, al, "memory") 387 388 #undef __LL_SC_CMPXCHG_DBL 389 #undef __CMPXCHG_DBL 390 391 #endif /* __ASM_ATOMIC_LSE_H */ 392