1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/atomic.h 4 * 5 * Copyright (C) 1996 Russell King. 6 * Copyright (C) 2002 Deep Blue Solutions Ltd. 7 * Copyright (C) 2012 ARM Ltd. 8 */ 9 10 #ifndef __ASM_ATOMIC_LSE_H 11 #define __ASM_ATOMIC_LSE_H 12 13 #define ATOMIC_OP(op, asm_op) \ 14 static inline void __lse_atomic_##op(int i, atomic_t *v) \ 15 { \ 16 asm volatile( \ 17 __LSE_PREAMBLE \ 18 " " #asm_op " %w[i], %[v]\n" \ 19 : [i] "+r" (i), [v] "+Q" (v->counter) \ 20 : "r" (v)); \ 21 } 22 23 ATOMIC_OP(andnot, stclr) 24 ATOMIC_OP(or, stset) 25 ATOMIC_OP(xor, steor) 26 ATOMIC_OP(add, stadd) 27 28 #undef ATOMIC_OP 29 30 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \ 31 static inline int __lse_atomic_fetch_##op##name(int i, atomic_t *v) \ 32 { \ 33 asm volatile( \ 34 __LSE_PREAMBLE \ 35 " " #asm_op #mb " %w[i], %w[i], %[v]" \ 36 : [i] "+r" (i), [v] "+Q" (v->counter) \ 37 : "r" (v) \ 38 : cl); \ 39 \ 40 return i; \ 41 } 42 43 #define ATOMIC_FETCH_OPS(op, asm_op) \ 44 ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \ 45 ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \ 46 ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \ 47 ATOMIC_FETCH_OP( , al, op, asm_op, "memory") 48 49 ATOMIC_FETCH_OPS(andnot, ldclr) 50 ATOMIC_FETCH_OPS(or, ldset) 51 ATOMIC_FETCH_OPS(xor, ldeor) 52 ATOMIC_FETCH_OPS(add, ldadd) 53 54 #undef ATOMIC_FETCH_OP 55 #undef ATOMIC_FETCH_OPS 56 57 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ 58 static inline int __lse_atomic_add_return##name(int i, atomic_t *v) \ 59 { \ 60 u32 tmp; \ 61 \ 62 asm volatile( \ 63 __LSE_PREAMBLE \ 64 " ldadd" #mb " %w[i], %w[tmp], %[v]\n" \ 65 " add %w[i], %w[i], %w[tmp]" \ 66 : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 67 : "r" (v) \ 68 : cl); \ 69 \ 70 return i; \ 71 } 72 73 ATOMIC_OP_ADD_RETURN(_relaxed, ) 74 ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") 75 ATOMIC_OP_ADD_RETURN(_release, l, "memory") 76 ATOMIC_OP_ADD_RETURN( , al, "memory") 77 78 #undef ATOMIC_OP_ADD_RETURN 79 80 static inline void __lse_atomic_and(int i, atomic_t *v) 81 { 82 asm volatile( 83 __LSE_PREAMBLE 84 " mvn %w[i], %w[i]\n" 85 " stclr %w[i], %[v]" 86 : [i] "+&r" (i), [v] "+Q" (v->counter) 87 : "r" (v)); 88 } 89 90 #define ATOMIC_FETCH_OP_AND(name, mb, cl...) \ 91 static inline int __lse_atomic_fetch_and##name(int i, atomic_t *v) \ 92 { \ 93 asm volatile( \ 94 __LSE_PREAMBLE \ 95 " mvn %w[i], %w[i]\n" \ 96 " ldclr" #mb " %w[i], %w[i], %[v]" \ 97 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 98 : "r" (v) \ 99 : cl); \ 100 \ 101 return i; \ 102 } 103 104 ATOMIC_FETCH_OP_AND(_relaxed, ) 105 ATOMIC_FETCH_OP_AND(_acquire, a, "memory") 106 ATOMIC_FETCH_OP_AND(_release, l, "memory") 107 ATOMIC_FETCH_OP_AND( , al, "memory") 108 109 #undef ATOMIC_FETCH_OP_AND 110 111 static inline void __lse_atomic_sub(int i, atomic_t *v) 112 { 113 asm volatile( 114 __LSE_PREAMBLE 115 " neg %w[i], %w[i]\n" 116 " stadd %w[i], %[v]" 117 : [i] "+&r" (i), [v] "+Q" (v->counter) 118 : "r" (v)); 119 } 120 121 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \ 122 static inline int __lse_atomic_sub_return##name(int i, atomic_t *v) \ 123 { \ 124 u32 tmp; \ 125 \ 126 asm volatile( \ 127 __LSE_PREAMBLE \ 128 " neg %w[i], %w[i]\n" \ 129 " ldadd" #mb " %w[i], %w[tmp], %[v]\n" \ 130 " add %w[i], %w[i], %w[tmp]" \ 131 : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 132 : "r" (v) \ 133 : cl); \ 134 \ 135 return i; \ 136 } 137 138 ATOMIC_OP_SUB_RETURN(_relaxed, ) 139 ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") 140 ATOMIC_OP_SUB_RETURN(_release, l, "memory") 141 ATOMIC_OP_SUB_RETURN( , al, "memory") 142 143 #undef ATOMIC_OP_SUB_RETURN 144 145 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \ 146 static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v) \ 147 { \ 148 asm volatile( \ 149 __LSE_PREAMBLE \ 150 " neg %w[i], %w[i]\n" \ 151 " ldadd" #mb " %w[i], %w[i], %[v]" \ 152 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 153 : "r" (v) \ 154 : cl); \ 155 \ 156 return i; \ 157 } 158 159 ATOMIC_FETCH_OP_SUB(_relaxed, ) 160 ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") 161 ATOMIC_FETCH_OP_SUB(_release, l, "memory") 162 ATOMIC_FETCH_OP_SUB( , al, "memory") 163 164 #undef ATOMIC_FETCH_OP_SUB 165 166 #define ATOMIC64_OP(op, asm_op) \ 167 static inline void __lse_atomic64_##op(s64 i, atomic64_t *v) \ 168 { \ 169 asm volatile( \ 170 __LSE_PREAMBLE \ 171 " " #asm_op " %[i], %[v]\n" \ 172 : [i] "+r" (i), [v] "+Q" (v->counter) \ 173 : "r" (v)); \ 174 } 175 176 ATOMIC64_OP(andnot, stclr) 177 ATOMIC64_OP(or, stset) 178 ATOMIC64_OP(xor, steor) 179 ATOMIC64_OP(add, stadd) 180 181 #undef ATOMIC64_OP 182 183 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \ 184 static inline long __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)\ 185 { \ 186 asm volatile( \ 187 __LSE_PREAMBLE \ 188 " " #asm_op #mb " %[i], %[i], %[v]" \ 189 : [i] "+r" (i), [v] "+Q" (v->counter) \ 190 : "r" (v) \ 191 : cl); \ 192 \ 193 return i; \ 194 } 195 196 #define ATOMIC64_FETCH_OPS(op, asm_op) \ 197 ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ 198 ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ 199 ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ 200 ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") 201 202 ATOMIC64_FETCH_OPS(andnot, ldclr) 203 ATOMIC64_FETCH_OPS(or, ldset) 204 ATOMIC64_FETCH_OPS(xor, ldeor) 205 ATOMIC64_FETCH_OPS(add, ldadd) 206 207 #undef ATOMIC64_FETCH_OP 208 #undef ATOMIC64_FETCH_OPS 209 210 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ 211 static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\ 212 { \ 213 unsigned long tmp; \ 214 \ 215 asm volatile( \ 216 __LSE_PREAMBLE \ 217 " ldadd" #mb " %[i], %x[tmp], %[v]\n" \ 218 " add %[i], %[i], %x[tmp]" \ 219 : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 220 : "r" (v) \ 221 : cl); \ 222 \ 223 return i; \ 224 } 225 226 ATOMIC64_OP_ADD_RETURN(_relaxed, ) 227 ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory") 228 ATOMIC64_OP_ADD_RETURN(_release, l, "memory") 229 ATOMIC64_OP_ADD_RETURN( , al, "memory") 230 231 #undef ATOMIC64_OP_ADD_RETURN 232 233 static inline void __lse_atomic64_and(s64 i, atomic64_t *v) 234 { 235 asm volatile( 236 __LSE_PREAMBLE 237 " mvn %[i], %[i]\n" 238 " stclr %[i], %[v]" 239 : [i] "+&r" (i), [v] "+Q" (v->counter) 240 : "r" (v)); 241 } 242 243 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \ 244 static inline long __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v) \ 245 { \ 246 asm volatile( \ 247 __LSE_PREAMBLE \ 248 " mvn %[i], %[i]\n" \ 249 " ldclr" #mb " %[i], %[i], %[v]" \ 250 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 251 : "r" (v) \ 252 : cl); \ 253 \ 254 return i; \ 255 } 256 257 ATOMIC64_FETCH_OP_AND(_relaxed, ) 258 ATOMIC64_FETCH_OP_AND(_acquire, a, "memory") 259 ATOMIC64_FETCH_OP_AND(_release, l, "memory") 260 ATOMIC64_FETCH_OP_AND( , al, "memory") 261 262 #undef ATOMIC64_FETCH_OP_AND 263 264 static inline void __lse_atomic64_sub(s64 i, atomic64_t *v) 265 { 266 asm volatile( 267 __LSE_PREAMBLE 268 " neg %[i], %[i]\n" 269 " stadd %[i], %[v]" 270 : [i] "+&r" (i), [v] "+Q" (v->counter) 271 : "r" (v)); 272 } 273 274 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \ 275 static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v) \ 276 { \ 277 unsigned long tmp; \ 278 \ 279 asm volatile( \ 280 __LSE_PREAMBLE \ 281 " neg %[i], %[i]\n" \ 282 " ldadd" #mb " %[i], %x[tmp], %[v]\n" \ 283 " add %[i], %[i], %x[tmp]" \ 284 : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 285 : "r" (v) \ 286 : cl); \ 287 \ 288 return i; \ 289 } 290 291 ATOMIC64_OP_SUB_RETURN(_relaxed, ) 292 ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") 293 ATOMIC64_OP_SUB_RETURN(_release, l, "memory") 294 ATOMIC64_OP_SUB_RETURN( , al, "memory") 295 296 #undef ATOMIC64_OP_SUB_RETURN 297 298 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \ 299 static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \ 300 { \ 301 asm volatile( \ 302 __LSE_PREAMBLE \ 303 " neg %[i], %[i]\n" \ 304 " ldadd" #mb " %[i], %[i], %[v]" \ 305 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 306 : "r" (v) \ 307 : cl); \ 308 \ 309 return i; \ 310 } 311 312 ATOMIC64_FETCH_OP_SUB(_relaxed, ) 313 ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory") 314 ATOMIC64_FETCH_OP_SUB(_release, l, "memory") 315 ATOMIC64_FETCH_OP_SUB( , al, "memory") 316 317 #undef ATOMIC64_FETCH_OP_SUB 318 319 static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v) 320 { 321 unsigned long tmp; 322 323 asm volatile( 324 __LSE_PREAMBLE 325 "1: ldr %x[tmp], %[v]\n" 326 " subs %[ret], %x[tmp], #1\n" 327 " b.lt 2f\n" 328 " casal %x[tmp], %[ret], %[v]\n" 329 " sub %x[tmp], %x[tmp], #1\n" 330 " sub %x[tmp], %x[tmp], %[ret]\n" 331 " cbnz %x[tmp], 1b\n" 332 "2:" 333 : [ret] "+&r" (v), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) 334 : 335 : "cc", "memory"); 336 337 return (long)v; 338 } 339 340 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \ 341 static __always_inline u##sz \ 342 __lse__cmpxchg_case_##name##sz(volatile void *ptr, \ 343 u##sz old, \ 344 u##sz new) \ 345 { \ 346 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ 347 register u##sz x1 asm ("x1") = old; \ 348 register u##sz x2 asm ("x2") = new; \ 349 unsigned long tmp; \ 350 \ 351 asm volatile( \ 352 __LSE_PREAMBLE \ 353 " mov %" #w "[tmp], %" #w "[old]\n" \ 354 " cas" #mb #sfx "\t%" #w "[tmp], %" #w "[new], %[v]\n" \ 355 " mov %" #w "[ret], %" #w "[tmp]" \ 356 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr), \ 357 [tmp] "=&r" (tmp) \ 358 : [old] "r" (x1), [new] "r" (x2) \ 359 : cl); \ 360 \ 361 return x0; \ 362 } 363 364 __CMPXCHG_CASE(w, b, , 8, ) 365 __CMPXCHG_CASE(w, h, , 16, ) 366 __CMPXCHG_CASE(w, , , 32, ) 367 __CMPXCHG_CASE(x, , , 64, ) 368 __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") 369 __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") 370 __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 371 __CMPXCHG_CASE(x, , acq_, 64, a, "memory") 372 __CMPXCHG_CASE(w, b, rel_, 8, l, "memory") 373 __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") 374 __CMPXCHG_CASE(w, , rel_, 32, l, "memory") 375 __CMPXCHG_CASE(x, , rel_, 64, l, "memory") 376 __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") 377 __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") 378 __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 379 __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 380 381 #undef __CMPXCHG_CASE 382 383 #define __CMPXCHG_DBL(name, mb, cl...) \ 384 static __always_inline long \ 385 __lse__cmpxchg_double##name(unsigned long old1, \ 386 unsigned long old2, \ 387 unsigned long new1, \ 388 unsigned long new2, \ 389 volatile void *ptr) \ 390 { \ 391 unsigned long oldval1 = old1; \ 392 unsigned long oldval2 = old2; \ 393 register unsigned long x0 asm ("x0") = old1; \ 394 register unsigned long x1 asm ("x1") = old2; \ 395 register unsigned long x2 asm ("x2") = new1; \ 396 register unsigned long x3 asm ("x3") = new2; \ 397 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ 398 \ 399 asm volatile( \ 400 __LSE_PREAMBLE \ 401 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ 402 " eor %[old1], %[old1], %[oldval1]\n" \ 403 " eor %[old2], %[old2], %[oldval2]\n" \ 404 " orr %[old1], %[old1], %[old2]" \ 405 : [old1] "+&r" (x0), [old2] "+&r" (x1), \ 406 [v] "+Q" (*(unsigned long *)ptr) \ 407 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ 408 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ 409 : cl); \ 410 \ 411 return x0; \ 412 } 413 414 __CMPXCHG_DBL( , ) 415 __CMPXCHG_DBL(_mb, al, "memory") 416 417 #undef __CMPXCHG_DBL 418 419 #endif /* __ASM_ATOMIC_LSE_H */ 420