1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #include <sys/atomic_common.h> 33 34 #define isb() __asm __volatile("isb" : : : "memory") 35 36 /* 37 * Options for DMB and DSB: 38 * oshld Outer Shareable, load 39 * oshst Outer Shareable, store 40 * osh Outer Shareable, all 41 * nshld Non-shareable, load 42 * nshst Non-shareable, store 43 * nsh Non-shareable, all 44 * ishld Inner Shareable, load 45 * ishst Inner Shareable, store 46 * ish Inner Shareable, all 47 * ld Full system, load 48 * st Full system, store 49 * sy Full system, all 50 */ 51 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 52 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 53 54 #define mb() dmb(sy) /* Full system memory barrier all */ 55 #define wmb() dmb(st) /* Full system memory barrier store */ 56 #define rmb() dmb(ld) /* Full system memory barrier load */ 57 58 #define ATOMIC_OP(op, asm_op, bar, a, l) \ 59 static __inline void \ 60 atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val) \ 61 { \ 62 uint32_t tmp; \ 63 int res; \ 64 \ 65 __asm __volatile( \ 66 "1: ld"#a"xr %w0, [%2] \n" \ 67 " "#asm_op" %w0, %w0, %w3 \n" \ 68 " st"#l"xr %w1, %w0, [%2] \n" \ 69 " cbnz %w1, 1b \n" \ 70 : "=&r"(tmp), "=&r"(res) \ 71 : "r" (p), "r" (val) \ 72 : "memory" \ 73 ); \ 74 } \ 75 \ 76 static __inline void \ 77 atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val) \ 78 { \ 79 uint64_t tmp; \ 80 int res; \ 81 \ 82 __asm __volatile( \ 83 "1: ld"#a"xr %0, [%2] \n" \ 84 " "#asm_op" %0, %0, %3 \n" \ 85 " st"#l"xr %w1, %0, [%2] \n" \ 86 " cbnz %w1, 1b \n" \ 87 : "=&r"(tmp), "=&r"(res) \ 88 : "r" (p), "r" (val) \ 89 : "memory" \ 90 ); \ 91 } 92 93 #define ATOMIC(op, asm_op) \ 94 ATOMIC_OP(op, asm_op, , , ) \ 95 ATOMIC_OP(op, asm_op, acq_, a, ) \ 96 ATOMIC_OP(op, asm_op, rel_, , l) \ 97 98 ATOMIC(add, add) 99 ATOMIC(clear, bic) 100 ATOMIC(set, orr) 101 ATOMIC(subtract, sub) 102 103 #define ATOMIC_FCMPSET(bar, a, l) \ 104 static __inline int \ 105 atomic_fcmpset_##bar##8(volatile uint8_t *p, uint8_t *cmpval, \ 106 uint8_t newval) \ 107 { \ 108 uint8_t tmp; \ 109 uint8_t _cmpval = *cmpval; \ 110 int res; \ 111 \ 112 __asm __volatile( \ 113 "1: mov %w1, #1 \n" \ 114 " ld"#a"xrb %w0, [%2] \n" \ 115 " cmp %w0, %w3 \n" \ 116 " b.ne 2f \n" \ 117 " st"#l"xrb %w1, %w4, [%2]\n" \ 118 "2:" \ 119 : "=&r"(tmp), "=&r"(res) \ 120 : "r" (p), "r" (_cmpval), "r" (newval) \ 121 : "cc", "memory" \ 122 ); \ 123 *cmpval = tmp; \ 124 \ 125 return (!res); \ 126 } \ 127 \ 128 static __inline int \ 129 atomic_fcmpset_##bar##16(volatile uint16_t *p, uint16_t *cmpval, \ 130 uint16_t newval) \ 131 { \ 132 uint16_t tmp; \ 133 uint16_t _cmpval = *cmpval; \ 134 int res; \ 135 \ 136 __asm __volatile( \ 137 "1: mov %w1, #1 \n" \ 138 " ld"#a"xh %w0, [%2] \n" \ 139 " cmp %w0, %w3 \n" \ 140 " b.ne 2f \n" \ 141 " st"#l"xh %w1, %w4, [%2] \n" \ 142 "2:" \ 143 : "=&r"(tmp), "=&r"(res) \ 144 : "r" (p), "r" (_cmpval), "r" (newval) \ 145 : "cc", "memory" \ 146 ); \ 147 *cmpval = tmp; \ 148 \ 149 return (!res); \ 150 } \ 151 \ 152 static __inline int \ 153 atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval, \ 154 uint32_t newval) \ 155 { \ 156 uint32_t tmp; \ 157 uint32_t _cmpval = *cmpval; \ 158 int res; \ 159 \ 160 __asm __volatile( \ 161 "1: mov %w1, #1 \n" \ 162 " ld"#a"xr %w0, [%2] \n" \ 163 " cmp %w0, %w3 \n" \ 164 " b.ne 2f \n" \ 165 " st"#l"xr %w1, %w4, [%2] \n" \ 166 "2:" \ 167 : "=&r"(tmp), "=&r"(res) \ 168 : "r" (p), "r" (_cmpval), "r" (newval) \ 169 : "cc", "memory" \ 170 ); \ 171 *cmpval = tmp; \ 172 \ 173 return (!res); \ 174 } \ 175 \ 176 static __inline int \ 177 atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval, \ 178 uint64_t newval) \ 179 { \ 180 uint64_t tmp; \ 181 uint64_t _cmpval = *cmpval; \ 182 int res; \ 183 \ 184 __asm __volatile( \ 185 "1: mov %w1, #1 \n" \ 186 " ld"#a"xr %0, [%2] \n" \ 187 " cmp %0, %3 \n" \ 188 " b.ne 2f \n" \ 189 " st"#l"xr %w1, %4, [%2] \n" \ 190 "2:" \ 191 : "=&r"(tmp), "=&r"(res) \ 192 : "r" (p), "r" (_cmpval), "r" (newval) \ 193 : "cc", "memory" \ 194 ); \ 195 *cmpval = tmp; \ 196 \ 197 return (!res); \ 198 } 199 200 ATOMIC_FCMPSET( , , ) 201 ATOMIC_FCMPSET(acq_, a, ) 202 ATOMIC_FCMPSET(rel_, ,l) 203 204 #undef ATOMIC_FCMPSET 205 206 #define ATOMIC_CMPSET(bar, a, l) \ 207 static __inline int \ 208 atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ 209 uint32_t newval) \ 210 { \ 211 uint32_t tmp; \ 212 int res; \ 213 \ 214 __asm __volatile( \ 215 "1: mov %w1, #1 \n" \ 216 " ld"#a"xr %w0, [%2] \n" \ 217 " cmp %w0, %w3 \n" \ 218 " b.ne 2f \n" \ 219 " st"#l"xr %w1, %w4, [%2] \n" \ 220 " cbnz %w1, 1b \n" \ 221 "2:" \ 222 : "=&r"(tmp), "=&r"(res) \ 223 : "r" (p), "r" (cmpval), "r" (newval) \ 224 : "cc", "memory" \ 225 ); \ 226 \ 227 return (!res); \ 228 } \ 229 \ 230 static __inline int \ 231 atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval, \ 232 uint64_t newval) \ 233 { \ 234 uint64_t tmp; \ 235 int res; \ 236 \ 237 __asm __volatile( \ 238 "1: mov %w1, #1 \n" \ 239 " ld"#a"xr %0, [%2] \n" \ 240 " cmp %0, %3 \n" \ 241 " b.ne 2f \n" \ 242 " st"#l"xr %w1, %4, [%2] \n" \ 243 " cbnz %w1, 1b \n" \ 244 "2:" \ 245 : "=&r"(tmp), "=&r"(res) \ 246 : "r" (p), "r" (cmpval), "r" (newval) \ 247 : "cc", "memory" \ 248 ); \ 249 \ 250 return (!res); \ 251 } 252 253 ATOMIC_CMPSET( , , ) 254 ATOMIC_CMPSET(acq_, a, ) 255 ATOMIC_CMPSET(rel_, ,l) 256 257 static __inline uint32_t 258 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 259 { 260 uint32_t tmp, ret; 261 int res; 262 263 __asm __volatile( 264 "1: ldxr %w2, [%3] \n" 265 " add %w0, %w2, %w4 \n" 266 " stxr %w1, %w0, [%3] \n" 267 " cbnz %w1, 1b \n" 268 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 269 : "r" (p), "r" (val) 270 : "memory" 271 ); 272 273 return (ret); 274 } 275 276 static __inline uint64_t 277 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 278 { 279 uint64_t tmp, ret; 280 int res; 281 282 __asm __volatile( 283 "1: ldxr %2, [%3] \n" 284 " add %0, %2, %4 \n" 285 " stxr %w1, %0, [%3] \n" 286 " cbnz %w1, 1b \n" 287 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 288 : "r" (p), "r" (val) 289 : "memory" 290 ); 291 292 return (ret); 293 } 294 295 static __inline uint32_t 296 atomic_readandclear_32(volatile uint32_t *p) 297 { 298 uint32_t ret; 299 int res; 300 301 __asm __volatile( 302 "1: ldxr %w1, [%2] \n" 303 " stxr %w0, wzr, [%2] \n" 304 " cbnz %w0, 1b \n" 305 : "=&r"(res), "=&r"(ret) 306 : "r" (p) 307 : "memory" 308 ); 309 310 return (ret); 311 } 312 313 static __inline uint64_t 314 atomic_readandclear_64(volatile uint64_t *p) 315 { 316 uint64_t ret; 317 int res; 318 319 __asm __volatile( 320 "1: ldxr %1, [%2] \n" 321 " stxr %w0, xzr, [%2] \n" 322 " cbnz %w0, 1b \n" 323 : "=&r"(res), "=&r"(ret) 324 : "r" (p) 325 : "memory" 326 ); 327 328 return (ret); 329 } 330 331 static __inline uint32_t 332 atomic_swap_32(volatile uint32_t *p, uint32_t val) 333 { 334 uint32_t ret; 335 int res; 336 337 __asm __volatile( 338 "1: ldxr %w0, [%2] \n" 339 " stxr %w1, %w3, [%2] \n" 340 " cbnz %w1, 1b \n" 341 : "=&r"(ret), "=&r"(res) 342 : "r" (p), "r" (val) 343 : "memory" 344 ); 345 346 return (ret); 347 } 348 349 static __inline uint64_t 350 atomic_swap_64(volatile uint64_t *p, uint64_t val) 351 { 352 uint64_t ret; 353 int res; 354 355 __asm __volatile( 356 "1: ldxr %0, [%2] \n" 357 " stxr %w1, %3, [%2] \n" 358 " cbnz %w1, 1b \n" 359 : "=&r"(ret), "=&r"(res) 360 : "r" (p), "r" (val) 361 : "memory" 362 ); 363 364 return (ret); 365 } 366 367 static __inline uint32_t 368 atomic_load_acq_32(volatile uint32_t *p) 369 { 370 uint32_t ret; 371 372 __asm __volatile( 373 "ldar %w0, [%1] \n" 374 : "=&r" (ret) 375 : "r" (p) 376 : "memory"); 377 378 return (ret); 379 } 380 381 static __inline uint64_t 382 atomic_load_acq_64(volatile uint64_t *p) 383 { 384 uint64_t ret; 385 386 __asm __volatile( 387 "ldar %0, [%1] \n" 388 : "=&r" (ret) 389 : "r" (p) 390 : "memory"); 391 392 return (ret); 393 } 394 395 static __inline void 396 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 397 { 398 399 __asm __volatile( 400 "stlr %w0, [%1] \n" 401 : 402 : "r" (val), "r" (p) 403 : "memory"); 404 } 405 406 static __inline void 407 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 408 { 409 410 __asm __volatile( 411 "stlr %0, [%1] \n" 412 : 413 : "r" (val), "r" (p) 414 : "memory"); 415 } 416 417 418 #define atomic_add_int atomic_add_32 419 #define atomic_fcmpset_int atomic_fcmpset_32 420 #define atomic_clear_int atomic_clear_32 421 #define atomic_cmpset_int atomic_cmpset_32 422 #define atomic_fetchadd_int atomic_fetchadd_32 423 #define atomic_readandclear_int atomic_readandclear_32 424 #define atomic_set_int atomic_set_32 425 #define atomic_swap_int atomic_swap_32 426 #define atomic_subtract_int atomic_subtract_32 427 428 #define atomic_add_acq_int atomic_add_acq_32 429 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 430 #define atomic_clear_acq_int atomic_clear_acq_32 431 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 432 #define atomic_load_acq_int atomic_load_acq_32 433 #define atomic_set_acq_int atomic_set_acq_32 434 #define atomic_subtract_acq_int atomic_subtract_acq_32 435 436 #define atomic_add_rel_int atomic_add_rel_32 437 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 438 #define atomic_clear_rel_int atomic_clear_rel_32 439 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 440 #define atomic_set_rel_int atomic_set_rel_32 441 #define atomic_subtract_rel_int atomic_subtract_rel_32 442 #define atomic_store_rel_int atomic_store_rel_32 443 444 #define atomic_add_long atomic_add_64 445 #define atomic_fcmpset_long atomic_fcmpset_64 446 #define atomic_clear_long atomic_clear_64 447 #define atomic_cmpset_long atomic_cmpset_64 448 #define atomic_fetchadd_long atomic_fetchadd_64 449 #define atomic_readandclear_long atomic_readandclear_64 450 #define atomic_set_long atomic_set_64 451 #define atomic_swap_long atomic_swap_64 452 #define atomic_subtract_long atomic_subtract_64 453 454 #define atomic_add_ptr atomic_add_64 455 #define atomic_fcmpset_ptr atomic_fcmpset_64 456 #define atomic_clear_ptr atomic_clear_64 457 #define atomic_cmpset_ptr atomic_cmpset_64 458 #define atomic_fetchadd_ptr atomic_fetchadd_64 459 #define atomic_readandclear_ptr atomic_readandclear_64 460 #define atomic_set_ptr atomic_set_64 461 #define atomic_swap_ptr atomic_swap_64 462 #define atomic_subtract_ptr atomic_subtract_64 463 464 #define atomic_add_acq_long atomic_add_acq_64 465 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 466 #define atomic_clear_acq_long atomic_clear_acq_64 467 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 468 #define atomic_load_acq_long atomic_load_acq_64 469 #define atomic_set_acq_long atomic_set_acq_64 470 #define atomic_subtract_acq_long atomic_subtract_acq_64 471 472 #define atomic_add_acq_ptr atomic_add_acq_64 473 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 474 #define atomic_clear_acq_ptr atomic_clear_acq_64 475 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 476 #define atomic_load_acq_ptr atomic_load_acq_64 477 #define atomic_set_acq_ptr atomic_set_acq_64 478 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 479 480 #define atomic_add_rel_long atomic_add_rel_64 481 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 482 #define atomic_clear_rel_long atomic_clear_rel_64 483 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 484 #define atomic_set_rel_long atomic_set_rel_64 485 #define atomic_subtract_rel_long atomic_subtract_rel_64 486 #define atomic_store_rel_long atomic_store_rel_64 487 488 #define atomic_add_rel_ptr atomic_add_rel_64 489 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 490 #define atomic_clear_rel_ptr atomic_clear_rel_64 491 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 492 #define atomic_set_rel_ptr atomic_set_rel_64 493 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 494 #define atomic_store_rel_ptr atomic_store_rel_64 495 496 static __inline void 497 atomic_thread_fence_acq(void) 498 { 499 500 dmb(ld); 501 } 502 503 static __inline void 504 atomic_thread_fence_rel(void) 505 { 506 507 dmb(sy); 508 } 509 510 static __inline void 511 atomic_thread_fence_acq_rel(void) 512 { 513 514 dmb(sy); 515 } 516 517 static __inline void 518 atomic_thread_fence_seq_cst(void) 519 { 520 521 dmb(sy); 522 } 523 524 #endif /* _MACHINE_ATOMIC_H_ */ 525 526