1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #define isb() __asm __volatile("isb" : : : "memory") 33 34 /* 35 * Options for DMB and DSB: 36 * oshld Outer Shareable, load 37 * oshst Outer Shareable, store 38 * osh Outer Shareable, all 39 * nshld Non-shareable, load 40 * nshst Non-shareable, store 41 * nsh Non-shareable, all 42 * ishld Inner Shareable, load 43 * ishst Inner Shareable, store 44 * ish Inner Shareable, all 45 * ld Full system, load 46 * st Full system, store 47 * sy Full system, all 48 */ 49 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 50 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 51 52 #define mb() dmb(sy) /* Full system memory barrier all */ 53 #define wmb() dmb(st) /* Full system memory barrier store */ 54 #define rmb() dmb(ld) /* Full system memory barrier load */ 55 56 #if defined(KCSAN) && !defined(KCSAN_RUNTIME) 57 #include <sys/_cscan_atomic.h> 58 #else 59 60 #include <sys/atomic_common.h> 61 62 #define ATOMIC_OP(op, asm_op, bar, a, l) \ 63 static __inline void \ 64 atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \ 65 { \ 66 uint8_t tmp; \ 67 int res; \ 68 \ 69 __asm __volatile( \ 70 "1: ld"#a"xrb %w0, [%2] \n" \ 71 " "#asm_op" %w0, %w0, %w3 \n" \ 72 " st"#l"xrb %w1, %w0, [%2] \n" \ 73 " cbnz %w1, 1b \n" \ 74 : "=&r"(tmp), "=&r"(res) \ 75 : "r" (p), "r" (val) \ 76 : "memory" \ 77 ); \ 78 } \ 79 \ 80 static __inline void \ 81 atomic_##op##_##bar##16(volatile uint16_t *p, uint16_t val) \ 82 { \ 83 uint16_t tmp; \ 84 int res; \ 85 \ 86 __asm __volatile( \ 87 "1: ld"#a"xrh %w0, [%2] \n" \ 88 " "#asm_op" %w0, %w0, %w3 \n" \ 89 " st"#l"xrh %w1, %w0, [%2] \n" \ 90 " cbnz %w1, 1b \n" \ 91 : "=&r"(tmp), "=&r"(res) \ 92 : "r" (p), "r" (val) \ 93 : "memory" \ 94 ); \ 95 } \ 96 \ 97 static __inline void \ 98 atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val) \ 99 { \ 100 uint32_t tmp; \ 101 int res; \ 102 \ 103 __asm __volatile( \ 104 "1: ld"#a"xr %w0, [%2] \n" \ 105 " "#asm_op" %w0, %w0, %w3 \n" \ 106 " st"#l"xr %w1, %w0, [%2] \n" \ 107 " cbnz %w1, 1b \n" \ 108 : "=&r"(tmp), "=&r"(res) \ 109 : "r" (p), "r" (val) \ 110 : "memory" \ 111 ); \ 112 } \ 113 \ 114 static __inline void \ 115 atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val) \ 116 { \ 117 uint64_t tmp; \ 118 int res; \ 119 \ 120 __asm __volatile( \ 121 "1: ld"#a"xr %0, [%2] \n" \ 122 " "#asm_op" %0, %0, %3 \n" \ 123 " st"#l"xr %w1, %0, [%2] \n" \ 124 " cbnz %w1, 1b \n" \ 125 : "=&r"(tmp), "=&r"(res) \ 126 : "r" (p), "r" (val) \ 127 : "memory" \ 128 ); \ 129 } 130 131 #define ATOMIC(op, asm_op) \ 132 ATOMIC_OP(op, asm_op, , , ) \ 133 ATOMIC_OP(op, asm_op, acq_, a, ) \ 134 ATOMIC_OP(op, asm_op, rel_, , l) \ 135 136 ATOMIC(add, add) 137 ATOMIC(clear, bic) 138 ATOMIC(set, orr) 139 ATOMIC(subtract, sub) 140 141 #define ATOMIC_FCMPSET(bar, a, l) \ 142 static __inline int \ 143 atomic_fcmpset_##bar##8(volatile uint8_t *p, uint8_t *cmpval, \ 144 uint8_t newval) \ 145 { \ 146 uint8_t tmp; \ 147 uint8_t _cmpval = *cmpval; \ 148 int res; \ 149 \ 150 __asm __volatile( \ 151 "1: mov %w1, #1 \n" \ 152 " ld"#a"xrb %w0, [%2] \n" \ 153 " cmp %w0, %w3 \n" \ 154 " b.ne 2f \n" \ 155 " st"#l"xrb %w1, %w4, [%2]\n" \ 156 "2:" \ 157 : "=&r"(tmp), "=&r"(res) \ 158 : "r" (p), "r" (_cmpval), "r" (newval) \ 159 : "cc", "memory" \ 160 ); \ 161 *cmpval = tmp; \ 162 \ 163 return (!res); \ 164 } \ 165 \ 166 static __inline int \ 167 atomic_fcmpset_##bar##16(volatile uint16_t *p, uint16_t *cmpval, \ 168 uint16_t newval) \ 169 { \ 170 uint16_t tmp; \ 171 uint16_t _cmpval = *cmpval; \ 172 int res; \ 173 \ 174 __asm __volatile( \ 175 "1: mov %w1, #1 \n" \ 176 " ld"#a"xrh %w0, [%2] \n" \ 177 " cmp %w0, %w3 \n" \ 178 " b.ne 2f \n" \ 179 " st"#l"xrh %w1, %w4, [%2] \n" \ 180 "2:" \ 181 : "=&r"(tmp), "=&r"(res) \ 182 : "r" (p), "r" (_cmpval), "r" (newval) \ 183 : "cc", "memory" \ 184 ); \ 185 *cmpval = tmp; \ 186 \ 187 return (!res); \ 188 } \ 189 \ 190 static __inline int \ 191 atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval, \ 192 uint32_t newval) \ 193 { \ 194 uint32_t tmp; \ 195 uint32_t _cmpval = *cmpval; \ 196 int res; \ 197 \ 198 __asm __volatile( \ 199 "1: mov %w1, #1 \n" \ 200 " ld"#a"xr %w0, [%2] \n" \ 201 " cmp %w0, %w3 \n" \ 202 " b.ne 2f \n" \ 203 " st"#l"xr %w1, %w4, [%2] \n" \ 204 "2:" \ 205 : "=&r"(tmp), "=&r"(res) \ 206 : "r" (p), "r" (_cmpval), "r" (newval) \ 207 : "cc", "memory" \ 208 ); \ 209 *cmpval = tmp; \ 210 \ 211 return (!res); \ 212 } \ 213 \ 214 static __inline int \ 215 atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval, \ 216 uint64_t newval) \ 217 { \ 218 uint64_t tmp; \ 219 uint64_t _cmpval = *cmpval; \ 220 int res; \ 221 \ 222 __asm __volatile( \ 223 "1: mov %w1, #1 \n" \ 224 " ld"#a"xr %0, [%2] \n" \ 225 " cmp %0, %3 \n" \ 226 " b.ne 2f \n" \ 227 " st"#l"xr %w1, %4, [%2] \n" \ 228 "2:" \ 229 : "=&r"(tmp), "=&r"(res) \ 230 : "r" (p), "r" (_cmpval), "r" (newval) \ 231 : "cc", "memory" \ 232 ); \ 233 *cmpval = tmp; \ 234 \ 235 return (!res); \ 236 } 237 238 ATOMIC_FCMPSET( , , ) 239 ATOMIC_FCMPSET(acq_, a, ) 240 ATOMIC_FCMPSET(rel_, ,l) 241 242 #undef ATOMIC_FCMPSET 243 244 #define ATOMIC_CMPSET(bar, a, l) \ 245 static __inline int \ 246 atomic_cmpset_##bar##8(volatile uint8_t *p, uint8_t cmpval, \ 247 uint8_t newval) \ 248 { \ 249 uint8_t tmp; \ 250 int res; \ 251 \ 252 __asm __volatile( \ 253 "1: mov %w1, #1 \n" \ 254 " ld"#a"xrb %w0, [%2] \n" \ 255 " cmp %w0, %w3 \n" \ 256 " b.ne 2f \n" \ 257 " st"#l"xrb %w1, %w4, [%2] \n" \ 258 " cbnz %w1, 1b \n" \ 259 "2:" \ 260 : "=&r"(tmp), "=&r"(res) \ 261 : "r" (p), "r" (cmpval), "r" (newval) \ 262 : "cc", "memory" \ 263 ); \ 264 \ 265 return (!res); \ 266 } \ 267 \ 268 static __inline int \ 269 atomic_cmpset_##bar##16(volatile uint16_t *p, uint16_t cmpval, \ 270 uint16_t newval) \ 271 { \ 272 uint16_t tmp; \ 273 int res; \ 274 \ 275 __asm __volatile( \ 276 "1: mov %w1, #1 \n" \ 277 " ld"#a"xrh %w0, [%2] \n" \ 278 " cmp %w0, %w3 \n" \ 279 " b.ne 2f \n" \ 280 " st"#l"xrh %w1, %w4, [%2] \n" \ 281 " cbnz %w1, 1b \n" \ 282 "2:" \ 283 : "=&r"(tmp), "=&r"(res) \ 284 : "r" (p), "r" (cmpval), "r" (newval) \ 285 : "cc", "memory" \ 286 ); \ 287 \ 288 return (!res); \ 289 } \ 290 \ 291 static __inline int \ 292 atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ 293 uint32_t newval) \ 294 { \ 295 uint32_t tmp; \ 296 int res; \ 297 \ 298 __asm __volatile( \ 299 "1: mov %w1, #1 \n" \ 300 " ld"#a"xr %w0, [%2] \n" \ 301 " cmp %w0, %w3 \n" \ 302 " b.ne 2f \n" \ 303 " st"#l"xr %w1, %w4, [%2] \n" \ 304 " cbnz %w1, 1b \n" \ 305 "2:" \ 306 : "=&r"(tmp), "=&r"(res) \ 307 : "r" (p), "r" (cmpval), "r" (newval) \ 308 : "cc", "memory" \ 309 ); \ 310 \ 311 return (!res); \ 312 } \ 313 \ 314 static __inline int \ 315 atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval, \ 316 uint64_t newval) \ 317 { \ 318 uint64_t tmp; \ 319 int res; \ 320 \ 321 __asm __volatile( \ 322 "1: mov %w1, #1 \n" \ 323 " ld"#a"xr %0, [%2] \n" \ 324 " cmp %0, %3 \n" \ 325 " b.ne 2f \n" \ 326 " st"#l"xr %w1, %4, [%2] \n" \ 327 " cbnz %w1, 1b \n" \ 328 "2:" \ 329 : "=&r"(tmp), "=&r"(res) \ 330 : "r" (p), "r" (cmpval), "r" (newval) \ 331 : "cc", "memory" \ 332 ); \ 333 \ 334 return (!res); \ 335 } 336 337 ATOMIC_CMPSET( , , ) 338 ATOMIC_CMPSET(acq_, a, ) 339 ATOMIC_CMPSET(rel_, ,l) 340 341 static __inline uint32_t 342 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 343 { 344 uint32_t tmp, ret; 345 int res; 346 347 __asm __volatile( 348 "1: ldxr %w2, [%3] \n" 349 " add %w0, %w2, %w4 \n" 350 " stxr %w1, %w0, [%3] \n" 351 " cbnz %w1, 1b \n" 352 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 353 : "r" (p), "r" (val) 354 : "memory" 355 ); 356 357 return (ret); 358 } 359 360 static __inline uint64_t 361 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 362 { 363 uint64_t tmp, ret; 364 int res; 365 366 __asm __volatile( 367 "1: ldxr %2, [%3] \n" 368 " add %0, %2, %4 \n" 369 " stxr %w1, %0, [%3] \n" 370 " cbnz %w1, 1b \n" 371 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 372 : "r" (p), "r" (val) 373 : "memory" 374 ); 375 376 return (ret); 377 } 378 379 static __inline uint32_t 380 atomic_readandclear_32(volatile uint32_t *p) 381 { 382 uint32_t ret; 383 int res; 384 385 __asm __volatile( 386 "1: ldxr %w1, [%2] \n" 387 " stxr %w0, wzr, [%2] \n" 388 " cbnz %w0, 1b \n" 389 : "=&r"(res), "=&r"(ret) 390 : "r" (p) 391 : "memory" 392 ); 393 394 return (ret); 395 } 396 397 static __inline uint64_t 398 atomic_readandclear_64(volatile uint64_t *p) 399 { 400 uint64_t ret; 401 int res; 402 403 __asm __volatile( 404 "1: ldxr %1, [%2] \n" 405 " stxr %w0, xzr, [%2] \n" 406 " cbnz %w0, 1b \n" 407 : "=&r"(res), "=&r"(ret) 408 : "r" (p) 409 : "memory" 410 ); 411 412 return (ret); 413 } 414 415 static __inline uint32_t 416 atomic_swap_32(volatile uint32_t *p, uint32_t val) 417 { 418 uint32_t ret; 419 int res; 420 421 __asm __volatile( 422 "1: ldxr %w0, [%2] \n" 423 " stxr %w1, %w3, [%2] \n" 424 " cbnz %w1, 1b \n" 425 : "=&r"(ret), "=&r"(res) 426 : "r" (p), "r" (val) 427 : "memory" 428 ); 429 430 return (ret); 431 } 432 433 static __inline uint64_t 434 atomic_swap_64(volatile uint64_t *p, uint64_t val) 435 { 436 uint64_t ret; 437 int res; 438 439 __asm __volatile( 440 "1: ldxr %0, [%2] \n" 441 " stxr %w1, %3, [%2] \n" 442 " cbnz %w1, 1b \n" 443 : "=&r"(ret), "=&r"(res) 444 : "r" (p), "r" (val) 445 : "memory" 446 ); 447 448 return (ret); 449 } 450 451 static __inline uint32_t 452 atomic_load_acq_32(volatile uint32_t *p) 453 { 454 uint32_t ret; 455 456 __asm __volatile( 457 "ldar %w0, [%1] \n" 458 : "=&r" (ret) 459 : "r" (p) 460 : "memory"); 461 462 return (ret); 463 } 464 465 static __inline uint64_t 466 atomic_load_acq_64(volatile uint64_t *p) 467 { 468 uint64_t ret; 469 470 __asm __volatile( 471 "ldar %0, [%1] \n" 472 : "=&r" (ret) 473 : "r" (p) 474 : "memory"); 475 476 return (ret); 477 } 478 479 static __inline void 480 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 481 { 482 483 __asm __volatile( 484 "stlr %w0, [%1] \n" 485 : 486 : "r" (val), "r" (p) 487 : "memory"); 488 } 489 490 static __inline void 491 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 492 { 493 494 __asm __volatile( 495 "stlr %0, [%1] \n" 496 : 497 : "r" (val), "r" (p) 498 : "memory"); 499 } 500 501 502 #define atomic_add_int atomic_add_32 503 #define atomic_fcmpset_int atomic_fcmpset_32 504 #define atomic_clear_int atomic_clear_32 505 #define atomic_cmpset_int atomic_cmpset_32 506 #define atomic_fetchadd_int atomic_fetchadd_32 507 #define atomic_readandclear_int atomic_readandclear_32 508 #define atomic_set_int atomic_set_32 509 #define atomic_swap_int atomic_swap_32 510 #define atomic_subtract_int atomic_subtract_32 511 512 #define atomic_add_acq_int atomic_add_acq_32 513 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 514 #define atomic_clear_acq_int atomic_clear_acq_32 515 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 516 #define atomic_load_acq_int atomic_load_acq_32 517 #define atomic_set_acq_int atomic_set_acq_32 518 #define atomic_subtract_acq_int atomic_subtract_acq_32 519 520 #define atomic_add_rel_int atomic_add_rel_32 521 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 522 #define atomic_clear_rel_int atomic_clear_rel_32 523 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 524 #define atomic_set_rel_int atomic_set_rel_32 525 #define atomic_subtract_rel_int atomic_subtract_rel_32 526 #define atomic_store_rel_int atomic_store_rel_32 527 528 #define atomic_add_long atomic_add_64 529 #define atomic_fcmpset_long atomic_fcmpset_64 530 #define atomic_clear_long atomic_clear_64 531 #define atomic_cmpset_long atomic_cmpset_64 532 #define atomic_fetchadd_long atomic_fetchadd_64 533 #define atomic_readandclear_long atomic_readandclear_64 534 #define atomic_set_long atomic_set_64 535 #define atomic_swap_long atomic_swap_64 536 #define atomic_subtract_long atomic_subtract_64 537 538 #define atomic_add_ptr atomic_add_64 539 #define atomic_fcmpset_ptr atomic_fcmpset_64 540 #define atomic_clear_ptr atomic_clear_64 541 #define atomic_cmpset_ptr atomic_cmpset_64 542 #define atomic_fetchadd_ptr atomic_fetchadd_64 543 #define atomic_readandclear_ptr atomic_readandclear_64 544 #define atomic_set_ptr atomic_set_64 545 #define atomic_swap_ptr atomic_swap_64 546 #define atomic_subtract_ptr atomic_subtract_64 547 548 #define atomic_add_acq_long atomic_add_acq_64 549 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 550 #define atomic_clear_acq_long atomic_clear_acq_64 551 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 552 #define atomic_load_acq_long atomic_load_acq_64 553 #define atomic_set_acq_long atomic_set_acq_64 554 #define atomic_subtract_acq_long atomic_subtract_acq_64 555 556 #define atomic_add_acq_ptr atomic_add_acq_64 557 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 558 #define atomic_clear_acq_ptr atomic_clear_acq_64 559 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 560 #define atomic_load_acq_ptr atomic_load_acq_64 561 #define atomic_set_acq_ptr atomic_set_acq_64 562 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 563 564 #define atomic_add_rel_long atomic_add_rel_64 565 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 566 #define atomic_clear_rel_long atomic_clear_rel_64 567 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 568 #define atomic_set_rel_long atomic_set_rel_64 569 #define atomic_subtract_rel_long atomic_subtract_rel_64 570 #define atomic_store_rel_long atomic_store_rel_64 571 572 #define atomic_add_rel_ptr atomic_add_rel_64 573 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 574 #define atomic_clear_rel_ptr atomic_clear_rel_64 575 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 576 #define atomic_set_rel_ptr atomic_set_rel_64 577 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 578 #define atomic_store_rel_ptr atomic_store_rel_64 579 580 static __inline void 581 atomic_thread_fence_acq(void) 582 { 583 584 dmb(ld); 585 } 586 587 static __inline void 588 atomic_thread_fence_rel(void) 589 { 590 591 dmb(sy); 592 } 593 594 static __inline void 595 atomic_thread_fence_acq_rel(void) 596 { 597 598 dmb(sy); 599 } 600 601 static __inline void 602 atomic_thread_fence_seq_cst(void) 603 { 604 605 dmb(sy); 606 } 607 608 #endif /* KCSAN && !KCSAN_RUNTIME */ 609 610 #endif /* _MACHINE_ATOMIC_H_ */ 611 612