1 /*- 2 * Copyright (c) 2015-2024 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #ifndef _MACHINE_ATOMIC_H_ 36 #define _MACHINE_ATOMIC_H_ 37 38 #include <sys/atomic_common.h> 39 40 #define fence() __asm __volatile("fence" ::: "memory"); 41 #define mb() fence() 42 #define rmb() fence() 43 #define wmb() fence() 44 45 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); 46 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); 47 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); 48 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *, 49 uint16_t); 50 51 #define ATOMIC_ACQ_REL(NAME, WIDTH) \ 52 static __inline void \ 53 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 54 { \ 55 atomic_##NAME##_##WIDTH(p, v); \ 56 fence(); \ 57 } \ 58 \ 59 static __inline void \ 60 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 61 { \ 62 fence(); \ 63 atomic_##NAME##_##WIDTH(p, v); \ 64 } 65 66 #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \ 67 static __inline int \ 68 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 69 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 70 { \ 71 int retval; \ 72 \ 73 retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \ 74 fence(); \ 75 return (retval); \ 76 } \ 77 \ 78 static __inline int \ 79 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 80 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 81 { \ 82 fence(); \ 83 return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \ 84 } 85 86 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \ 87 static __inline int \ 88 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 89 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 90 { \ 91 int retval; \ 92 \ 93 retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \ 94 fence(); \ 95 return (retval); \ 96 } \ 97 \ 98 static __inline int \ 99 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 100 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 101 { \ 102 fence(); \ 103 return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \ 104 } 105 106 ATOMIC_CMPSET_ACQ_REL(8); 107 ATOMIC_FCMPSET_ACQ_REL(8); 108 109 #define atomic_load_acq_char atomic_load_acq_8 110 #define atomic_cmpset_char atomic_cmpset_8 111 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 112 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 113 #define atomic_fcmpset_char atomic_fcmpset_8 114 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 115 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 116 117 #define atomic_cmpset_short atomic_cmpset_16 118 #define atomic_fcmpset_short atomic_fcmpset_16 119 120 ATOMIC_CMPSET_ACQ_REL(16); 121 ATOMIC_FCMPSET_ACQ_REL(16); 122 123 #define atomic_load_acq_16 atomic_load_acq_16 124 static __inline uint16_t 125 atomic_load_acq_16(const volatile uint16_t *p) 126 { 127 uint16_t ret; 128 129 ret = *p; 130 131 fence(); 132 133 return (ret); 134 } 135 136 static __inline void 137 atomic_store_rel_16(volatile uint16_t *p, uint16_t val) 138 { 139 140 fence(); 141 142 *p = val; 143 } 144 145 #define atomic_cmpset_acq_short atomic_cmpset_acq_16 146 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 147 #define atomic_load_acq_short atomic_load_acq_16 148 149 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 150 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 151 #define atomic_store_rel_short atomic_store_rel_16 152 153 static __inline void 154 atomic_add_32(volatile uint32_t *p, uint32_t val) 155 { 156 157 __asm __volatile("amoadd.w zero, %1, %0" 158 : "+A" (*p) 159 : "r" (val) 160 : "memory"); 161 } 162 163 static __inline void 164 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 165 { 166 167 __asm __volatile("amoadd.w zero, %1, %0" 168 : "+A" (*p) 169 : "r" (-val) 170 : "memory"); 171 } 172 173 static __inline void 174 atomic_set_32(volatile uint32_t *p, uint32_t val) 175 { 176 177 __asm __volatile("amoor.w zero, %1, %0" 178 : "+A" (*p) 179 : "r" (val) 180 : "memory"); 181 } 182 183 static __inline void 184 atomic_clear_32(volatile uint32_t *p, uint32_t val) 185 { 186 187 __asm __volatile("amoand.w zero, %1, %0" 188 : "+A" (*p) 189 : "r" (~val) 190 : "memory"); 191 } 192 193 static __inline int 194 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 195 { 196 uint32_t tmp; 197 int res; 198 199 res = 0; 200 201 __asm __volatile( 202 "0:" 203 "li %1, 1\n" /* Preset to fail */ 204 "lr.w %0, %2\n" 205 "bne %0, %z3, 1f\n" 206 "sc.w %1, %z4, %2\n" 207 "bnez %1, 0b\n" 208 "1:" 209 : "=&r" (tmp), "=&r" (res), "+A" (*p) 210 : "rJ" ((long)(int32_t)cmpval), "rJ" (newval) 211 : "memory"); 212 213 return (!res); 214 } 215 216 static __inline int 217 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 218 { 219 uint32_t tmp; 220 int res; 221 222 res = 0; 223 224 __asm __volatile( 225 "0:" 226 "li %1, 1\n" /* Preset to fail */ 227 "lr.w %0, %2\n" /* Load old value */ 228 "bne %0, %z4, 1f\n" /* Compare */ 229 "sc.w %1, %z5, %2\n" /* Try to store new value */ 230 "j 2f\n" 231 "1:" 232 "sw %0, %3\n" /* Save old value */ 233 "2:" 234 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 235 : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval) 236 : "memory"); 237 238 return (!res); 239 } 240 241 static __inline uint32_t 242 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 243 { 244 uint32_t ret; 245 246 __asm __volatile("amoadd.w %0, %2, %1" 247 : "=&r" (ret), "+A" (*p) 248 : "r" (val) 249 : "memory"); 250 251 return (ret); 252 } 253 254 static __inline uint32_t 255 atomic_readandclear_32(volatile uint32_t *p) 256 { 257 uint32_t ret; 258 uint32_t val; 259 260 val = 0; 261 262 __asm __volatile("amoswap.w %0, %2, %1" 263 : "=&r"(ret), "+A" (*p) 264 : "r" (val) 265 : "memory"); 266 267 return (ret); 268 } 269 270 static __inline int 271 atomic_testandclear_32(volatile uint32_t *p, u_int val) 272 { 273 uint32_t mask, old; 274 275 mask = 1u << (val & 31); 276 __asm __volatile("amoand.w %0, %2, %1" 277 : "=&r" (old), "+A" (*p) 278 : "r" (~mask) 279 : "memory"); 280 281 return ((old & mask) != 0); 282 } 283 284 static __inline int 285 atomic_testandset_32(volatile uint32_t *p, u_int val) 286 { 287 uint32_t mask, old; 288 289 mask = 1u << (val & 31); 290 __asm __volatile("amoor.w %0, %2, %1" 291 : "=&r" (old), "+A" (*p) 292 : "r" (mask) 293 : "memory"); 294 295 return ((old & mask) != 0); 296 } 297 298 #define atomic_add_int atomic_add_32 299 #define atomic_clear_int atomic_clear_32 300 #define atomic_cmpset_int atomic_cmpset_32 301 #define atomic_fcmpset_int atomic_fcmpset_32 302 #define atomic_fetchadd_int atomic_fetchadd_32 303 #define atomic_readandclear_int atomic_readandclear_32 304 #define atomic_set_int atomic_set_32 305 #define atomic_subtract_int atomic_subtract_32 306 #define atomic_testandclear_int atomic_testandclear_32 307 #define atomic_testandset_int atomic_testandset_32 308 309 ATOMIC_ACQ_REL(set, 32) 310 ATOMIC_ACQ_REL(clear, 32) 311 ATOMIC_ACQ_REL(add, 32) 312 ATOMIC_ACQ_REL(subtract, 32) 313 314 ATOMIC_CMPSET_ACQ_REL(32); 315 ATOMIC_FCMPSET_ACQ_REL(32); 316 317 static __inline uint32_t 318 atomic_load_acq_32(const volatile uint32_t *p) 319 { 320 uint32_t ret; 321 322 ret = *p; 323 324 fence(); 325 326 return (ret); 327 } 328 329 static __inline void 330 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 331 { 332 333 fence(); 334 335 *p = val; 336 } 337 338 #define atomic_add_acq_int atomic_add_acq_32 339 #define atomic_clear_acq_int atomic_clear_acq_32 340 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 341 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 342 #define atomic_load_acq_int atomic_load_acq_32 343 #define atomic_set_acq_int atomic_set_acq_32 344 #define atomic_subtract_acq_int atomic_subtract_acq_32 345 346 #define atomic_add_rel_int atomic_add_rel_32 347 #define atomic_clear_rel_int atomic_clear_rel_32 348 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 349 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 350 #define atomic_set_rel_int atomic_set_rel_32 351 #define atomic_subtract_rel_int atomic_subtract_rel_32 352 #define atomic_store_rel_int atomic_store_rel_32 353 354 static __inline void 355 atomic_add_64(volatile uint64_t *p, uint64_t val) 356 { 357 358 __asm __volatile("amoadd.d zero, %1, %0" 359 : "+A" (*p) 360 : "r" (val) 361 : "memory"); 362 } 363 364 static __inline void 365 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 366 { 367 368 __asm __volatile("amoadd.d zero, %1, %0" 369 : "+A" (*p) 370 : "r" (-val) 371 : "memory"); 372 } 373 374 static __inline void 375 atomic_set_64(volatile uint64_t *p, uint64_t val) 376 { 377 378 __asm __volatile("amoor.d zero, %1, %0" 379 : "+A" (*p) 380 : "r" (val) 381 : "memory"); 382 } 383 384 static __inline void 385 atomic_clear_64(volatile uint64_t *p, uint64_t val) 386 { 387 388 __asm __volatile("amoand.d zero, %1, %0" 389 : "+A" (*p) 390 : "r" (~val) 391 : "memory"); 392 } 393 394 static __inline int 395 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 396 { 397 uint64_t tmp; 398 int res; 399 400 res = 0; 401 402 __asm __volatile( 403 "0:" 404 "li %1, 1\n" /* Preset to fail */ 405 "lr.d %0, %2\n" 406 "bne %0, %z3, 1f\n" 407 "sc.d %1, %z4, %2\n" 408 "bnez %1, 0b\n" 409 "1:" 410 : "=&r" (tmp), "=&r" (res), "+A" (*p) 411 : "rJ" (cmpval), "rJ" (newval) 412 : "memory"); 413 414 return (!res); 415 } 416 417 static __inline int 418 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 419 { 420 uint64_t tmp; 421 int res; 422 423 res = 0; 424 425 __asm __volatile( 426 "0:" 427 "li %1, 1\n" /* Preset to fail */ 428 "lr.d %0, %2\n" /* Load old value */ 429 "bne %0, %z4, 1f\n" /* Compare */ 430 "sc.d %1, %z5, %2\n" /* Try to store new value */ 431 "j 2f\n" 432 "1:" 433 "sd %0, %3\n" /* Save old value */ 434 "2:" 435 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 436 : "rJ" (*cmpval), "rJ" (newval) 437 : "memory"); 438 439 return (!res); 440 } 441 442 static __inline uint64_t 443 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 444 { 445 uint64_t ret; 446 447 __asm __volatile("amoadd.d %0, %2, %1" 448 : "=&r" (ret), "+A" (*p) 449 : "r" (val) 450 : "memory"); 451 452 return (ret); 453 } 454 455 static __inline uint64_t 456 atomic_readandclear_64(volatile uint64_t *p) 457 { 458 uint64_t ret; 459 uint64_t val; 460 461 val = 0; 462 463 __asm __volatile("amoswap.d %0, %2, %1" 464 : "=&r"(ret), "+A" (*p) 465 : "r" (val) 466 : "memory"); 467 468 return (ret); 469 } 470 471 static __inline int 472 atomic_testandclear_64(volatile uint64_t *p, u_int val) 473 { 474 uint64_t mask, old; 475 476 mask = 1ul << (val & 63); 477 __asm __volatile("amoand.d %0, %2, %1" 478 : "=&r" (old), "+A" (*p) 479 : "r" (~mask) 480 : "memory"); 481 482 return ((old & mask) != 0); 483 } 484 485 static __inline int 486 atomic_testandset_64(volatile uint64_t *p, u_int val) 487 { 488 uint64_t mask, old; 489 490 mask = 1ul << (val & 63); 491 __asm __volatile("amoor.d %0, %2, %1" 492 : "=&r" (old), "+A" (*p) 493 : "r" (mask) 494 : "memory"); 495 496 return ((old & mask) != 0); 497 } 498 499 static __inline int 500 atomic_testandset_acq_64(volatile uint64_t *p, u_int val) 501 { 502 uint64_t mask, old; 503 504 mask = 1ul << (val & 63); 505 __asm __volatile("amoor.d.aq %0, %2, %1" 506 : "=&r" (old), "+A" (*p) 507 : "r" (mask) 508 : "memory"); 509 510 return ((old & mask) != 0); 511 } 512 513 static __inline uint32_t 514 atomic_swap_32(volatile uint32_t *p, uint32_t val) 515 { 516 uint32_t old; 517 518 __asm __volatile("amoswap.w %0, %2, %1" 519 : "=&r"(old), "+A" (*p) 520 : "r" (val) 521 : "memory"); 522 523 return (old); 524 } 525 526 static __inline uint64_t 527 atomic_swap_64(volatile uint64_t *p, uint64_t val) 528 { 529 uint64_t old; 530 531 __asm __volatile("amoswap.d %0, %2, %1" 532 : "=&r"(old), "+A" (*p) 533 : "r" (val) 534 : "memory"); 535 536 return (old); 537 } 538 539 #define atomic_swap_int atomic_swap_32 540 541 #define atomic_add_long atomic_add_64 542 #define atomic_clear_long atomic_clear_64 543 #define atomic_cmpset_long atomic_cmpset_64 544 #define atomic_fcmpset_long atomic_fcmpset_64 545 #define atomic_fetchadd_long atomic_fetchadd_64 546 #define atomic_readandclear_long atomic_readandclear_64 547 #define atomic_set_long atomic_set_64 548 #define atomic_subtract_long atomic_subtract_64 549 #define atomic_swap_long atomic_swap_64 550 #define atomic_testandclear_long atomic_testandclear_64 551 #define atomic_testandset_long atomic_testandset_64 552 #define atomic_testandset_acq_long atomic_testandset_acq_64 553 554 #define atomic_add_ptr atomic_add_64 555 #define atomic_clear_ptr atomic_clear_64 556 #define atomic_cmpset_ptr atomic_cmpset_64 557 #define atomic_fcmpset_ptr atomic_fcmpset_64 558 #define atomic_fetchadd_ptr atomic_fetchadd_64 559 #define atomic_readandclear_ptr atomic_readandclear_64 560 #define atomic_set_ptr atomic_set_64 561 #define atomic_subtract_ptr atomic_subtract_64 562 #define atomic_swap_ptr atomic_swap_64 563 #define atomic_testandclear_ptr atomic_testandclear_64 564 #define atomic_testandset_ptr atomic_testandset_64 565 566 ATOMIC_ACQ_REL(set, 64) 567 ATOMIC_ACQ_REL(clear, 64) 568 ATOMIC_ACQ_REL(add, 64) 569 ATOMIC_ACQ_REL(subtract, 64) 570 571 ATOMIC_CMPSET_ACQ_REL(64); 572 ATOMIC_FCMPSET_ACQ_REL(64); 573 574 static __inline uint64_t 575 atomic_load_acq_64(const volatile uint64_t *p) 576 { 577 uint64_t ret; 578 579 ret = *p; 580 581 fence(); 582 583 return (ret); 584 } 585 586 static __inline void 587 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 588 { 589 590 fence(); 591 592 *p = val; 593 } 594 595 #define atomic_add_acq_long atomic_add_acq_64 596 #define atomic_clear_acq_long atomic_clear_acq_64 597 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 598 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 599 #define atomic_load_acq_long atomic_load_acq_64 600 #define atomic_set_acq_long atomic_set_acq_64 601 #define atomic_subtract_acq_long atomic_subtract_acq_64 602 603 #define atomic_add_acq_ptr atomic_add_acq_64 604 #define atomic_clear_acq_ptr atomic_clear_acq_64 605 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 606 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 607 #define atomic_load_acq_ptr atomic_load_acq_64 608 #define atomic_set_acq_ptr atomic_set_acq_64 609 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 610 611 #undef ATOMIC_ACQ_REL 612 613 static __inline void 614 atomic_thread_fence_acq(void) 615 { 616 617 fence(); 618 } 619 620 static __inline void 621 atomic_thread_fence_rel(void) 622 { 623 624 fence(); 625 } 626 627 static __inline void 628 atomic_thread_fence_acq_rel(void) 629 { 630 631 fence(); 632 } 633 634 static __inline void 635 atomic_thread_fence_seq_cst(void) 636 { 637 638 fence(); 639 } 640 641 #define atomic_add_rel_long atomic_add_rel_64 642 #define atomic_clear_rel_long atomic_clear_rel_64 643 644 #define atomic_add_rel_long atomic_add_rel_64 645 #define atomic_clear_rel_long atomic_clear_rel_64 646 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 647 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 648 #define atomic_set_rel_long atomic_set_rel_64 649 #define atomic_subtract_rel_long atomic_subtract_rel_64 650 #define atomic_store_rel_long atomic_store_rel_64 651 652 #define atomic_add_rel_ptr atomic_add_rel_64 653 #define atomic_clear_rel_ptr atomic_clear_rel_64 654 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 655 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 656 #define atomic_set_rel_ptr atomic_set_rel_64 657 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 658 #define atomic_store_rel_ptr atomic_store_rel_64 659 660 #include <sys/_atomic_subword.h> 661 662 #define atomic_set_short atomic_set_16 663 #define atomic_clear_short atomic_clear_16 664 665 #endif /* _MACHINE_ATOMIC_H_ */ 666