1 /*- 2 * Copyright (c) 2015-2024 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #ifndef _MACHINE_ATOMIC_H_ 36 #define _MACHINE_ATOMIC_H_ 37 38 #include <sys/atomic_common.h> 39 40 #define fence() __asm __volatile("fence" ::: "memory"); 41 #define mb() fence() 42 #define rmb() fence() 43 #define wmb() fence() 44 45 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); 46 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); 47 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); 48 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *, 49 uint16_t); 50 51 #define ATOMIC_ACQ_REL(NAME, WIDTH) \ 52 static __inline void \ 53 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 54 { \ 55 atomic_##NAME##_##WIDTH(p, v); \ 56 fence(); \ 57 } \ 58 \ 59 static __inline void \ 60 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 61 { \ 62 fence(); \ 63 atomic_##NAME##_##WIDTH(p, v); \ 64 } 65 66 #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \ 67 static __inline int \ 68 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 69 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 70 { \ 71 int retval; \ 72 \ 73 retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \ 74 fence(); \ 75 return (retval); \ 76 } \ 77 \ 78 static __inline int \ 79 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 80 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 81 { \ 82 fence(); \ 83 return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \ 84 } 85 86 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \ 87 static __inline int \ 88 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 89 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 90 { \ 91 int retval; \ 92 \ 93 retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \ 94 fence(); \ 95 return (retval); \ 96 } \ 97 \ 98 static __inline int \ 99 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 100 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 101 { \ 102 fence(); \ 103 return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \ 104 } 105 106 ATOMIC_CMPSET_ACQ_REL(8); 107 ATOMIC_FCMPSET_ACQ_REL(8); 108 109 #define atomic_cmpset_char atomic_cmpset_8 110 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 111 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 112 #define atomic_fcmpset_char atomic_fcmpset_8 113 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 114 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 115 116 #define atomic_cmpset_short atomic_cmpset_16 117 #define atomic_fcmpset_short atomic_fcmpset_16 118 119 ATOMIC_CMPSET_ACQ_REL(16); 120 ATOMIC_FCMPSET_ACQ_REL(16); 121 122 #define atomic_load_acq_16 atomic_load_acq_16 123 static __inline uint16_t 124 atomic_load_acq_16(volatile uint16_t *p) 125 { 126 uint16_t ret; 127 128 ret = *p; 129 130 fence(); 131 132 return (ret); 133 } 134 135 static __inline void 136 atomic_store_rel_16(volatile uint16_t *p, uint16_t val) 137 { 138 139 fence(); 140 141 *p = val; 142 } 143 144 #define atomic_cmpset_acq_short atomic_cmpset_acq_16 145 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 146 #define atomic_load_acq_short atomic_load_acq_16 147 148 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 149 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 150 #define atomic_store_rel_short atomic_store_rel_16 151 152 static __inline void 153 atomic_add_32(volatile uint32_t *p, uint32_t val) 154 { 155 156 __asm __volatile("amoadd.w zero, %1, %0" 157 : "+A" (*p) 158 : "r" (val) 159 : "memory"); 160 } 161 162 static __inline void 163 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 164 { 165 166 __asm __volatile("amoadd.w zero, %1, %0" 167 : "+A" (*p) 168 : "r" (-val) 169 : "memory"); 170 } 171 172 static __inline void 173 atomic_set_32(volatile uint32_t *p, uint32_t val) 174 { 175 176 __asm __volatile("amoor.w zero, %1, %0" 177 : "+A" (*p) 178 : "r" (val) 179 : "memory"); 180 } 181 182 static __inline void 183 atomic_clear_32(volatile uint32_t *p, uint32_t val) 184 { 185 186 __asm __volatile("amoand.w zero, %1, %0" 187 : "+A" (*p) 188 : "r" (~val) 189 : "memory"); 190 } 191 192 static __inline int 193 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 194 { 195 uint32_t tmp; 196 int res; 197 198 res = 0; 199 200 __asm __volatile( 201 "0:" 202 "li %1, 1\n" /* Preset to fail */ 203 "lr.w %0, %2\n" 204 "bne %0, %z3, 1f\n" 205 "sc.w %1, %z4, %2\n" 206 "bnez %1, 0b\n" 207 "1:" 208 : "=&r" (tmp), "=&r" (res), "+A" (*p) 209 : "rJ" ((long)(int32_t)cmpval), "rJ" (newval) 210 : "memory"); 211 212 return (!res); 213 } 214 215 static __inline int 216 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 217 { 218 uint32_t tmp; 219 int res; 220 221 res = 0; 222 223 __asm __volatile( 224 "0:" 225 "li %1, 1\n" /* Preset to fail */ 226 "lr.w %0, %2\n" /* Load old value */ 227 "bne %0, %z4, 1f\n" /* Compare */ 228 "sc.w %1, %z5, %2\n" /* Try to store new value */ 229 "j 2f\n" 230 "1:" 231 "sw %0, %3\n" /* Save old value */ 232 "2:" 233 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 234 : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval) 235 : "memory"); 236 237 return (!res); 238 } 239 240 static __inline uint32_t 241 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 242 { 243 uint32_t ret; 244 245 __asm __volatile("amoadd.w %0, %2, %1" 246 : "=&r" (ret), "+A" (*p) 247 : "r" (val) 248 : "memory"); 249 250 return (ret); 251 } 252 253 static __inline uint32_t 254 atomic_readandclear_32(volatile uint32_t *p) 255 { 256 uint32_t ret; 257 uint32_t val; 258 259 val = 0; 260 261 __asm __volatile("amoswap.w %0, %2, %1" 262 : "=&r"(ret), "+A" (*p) 263 : "r" (val) 264 : "memory"); 265 266 return (ret); 267 } 268 269 #define atomic_add_int atomic_add_32 270 #define atomic_clear_int atomic_clear_32 271 #define atomic_cmpset_int atomic_cmpset_32 272 #define atomic_fcmpset_int atomic_fcmpset_32 273 #define atomic_fetchadd_int atomic_fetchadd_32 274 #define atomic_readandclear_int atomic_readandclear_32 275 #define atomic_set_int atomic_set_32 276 #define atomic_subtract_int atomic_subtract_32 277 278 ATOMIC_ACQ_REL(set, 32) 279 ATOMIC_ACQ_REL(clear, 32) 280 ATOMIC_ACQ_REL(add, 32) 281 ATOMIC_ACQ_REL(subtract, 32) 282 283 ATOMIC_CMPSET_ACQ_REL(32); 284 ATOMIC_FCMPSET_ACQ_REL(32); 285 286 static __inline uint32_t 287 atomic_load_acq_32(volatile uint32_t *p) 288 { 289 uint32_t ret; 290 291 ret = *p; 292 293 fence(); 294 295 return (ret); 296 } 297 298 static __inline void 299 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 300 { 301 302 fence(); 303 304 *p = val; 305 } 306 307 #define atomic_add_acq_int atomic_add_acq_32 308 #define atomic_clear_acq_int atomic_clear_acq_32 309 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 310 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 311 #define atomic_load_acq_int atomic_load_acq_32 312 #define atomic_set_acq_int atomic_set_acq_32 313 #define atomic_subtract_acq_int atomic_subtract_acq_32 314 315 #define atomic_add_rel_int atomic_add_rel_32 316 #define atomic_clear_rel_int atomic_clear_rel_32 317 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 318 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 319 #define atomic_set_rel_int atomic_set_rel_32 320 #define atomic_subtract_rel_int atomic_subtract_rel_32 321 #define atomic_store_rel_int atomic_store_rel_32 322 323 static __inline void 324 atomic_add_64(volatile uint64_t *p, uint64_t val) 325 { 326 327 __asm __volatile("amoadd.d zero, %1, %0" 328 : "+A" (*p) 329 : "r" (val) 330 : "memory"); 331 } 332 333 static __inline void 334 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 335 { 336 337 __asm __volatile("amoadd.d zero, %1, %0" 338 : "+A" (*p) 339 : "r" (-val) 340 : "memory"); 341 } 342 343 static __inline void 344 atomic_set_64(volatile uint64_t *p, uint64_t val) 345 { 346 347 __asm __volatile("amoor.d zero, %1, %0" 348 : "+A" (*p) 349 : "r" (val) 350 : "memory"); 351 } 352 353 static __inline void 354 atomic_clear_64(volatile uint64_t *p, uint64_t val) 355 { 356 357 __asm __volatile("amoand.d zero, %1, %0" 358 : "+A" (*p) 359 : "r" (~val) 360 : "memory"); 361 } 362 363 static __inline int 364 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 365 { 366 uint64_t tmp; 367 int res; 368 369 res = 0; 370 371 __asm __volatile( 372 "0:" 373 "li %1, 1\n" /* Preset to fail */ 374 "lr.d %0, %2\n" 375 "bne %0, %z3, 1f\n" 376 "sc.d %1, %z4, %2\n" 377 "bnez %1, 0b\n" 378 "1:" 379 : "=&r" (tmp), "=&r" (res), "+A" (*p) 380 : "rJ" (cmpval), "rJ" (newval) 381 : "memory"); 382 383 return (!res); 384 } 385 386 static __inline int 387 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 388 { 389 uint64_t tmp; 390 int res; 391 392 res = 0; 393 394 __asm __volatile( 395 "0:" 396 "li %1, 1\n" /* Preset to fail */ 397 "lr.d %0, %2\n" /* Load old value */ 398 "bne %0, %z4, 1f\n" /* Compare */ 399 "sc.d %1, %z5, %2\n" /* Try to store new value */ 400 "j 2f\n" 401 "1:" 402 "sd %0, %3\n" /* Save old value */ 403 "2:" 404 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 405 : "rJ" (*cmpval), "rJ" (newval) 406 : "memory"); 407 408 return (!res); 409 } 410 411 static __inline uint64_t 412 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 413 { 414 uint64_t ret; 415 416 __asm __volatile("amoadd.d %0, %2, %1" 417 : "=&r" (ret), "+A" (*p) 418 : "r" (val) 419 : "memory"); 420 421 return (ret); 422 } 423 424 static __inline uint64_t 425 atomic_readandclear_64(volatile uint64_t *p) 426 { 427 uint64_t ret; 428 uint64_t val; 429 430 val = 0; 431 432 __asm __volatile("amoswap.d %0, %2, %1" 433 : "=&r"(ret), "+A" (*p) 434 : "r" (val) 435 : "memory"); 436 437 return (ret); 438 } 439 440 static __inline uint32_t 441 atomic_swap_32(volatile uint32_t *p, uint32_t val) 442 { 443 uint32_t old; 444 445 __asm __volatile("amoswap.w %0, %2, %1" 446 : "=&r"(old), "+A" (*p) 447 : "r" (val) 448 : "memory"); 449 450 return (old); 451 } 452 453 static __inline uint64_t 454 atomic_swap_64(volatile uint64_t *p, uint64_t val) 455 { 456 uint64_t old; 457 458 __asm __volatile("amoswap.d %0, %2, %1" 459 : "=&r"(old), "+A" (*p) 460 : "r" (val) 461 : "memory"); 462 463 return (old); 464 } 465 466 #define atomic_swap_int atomic_swap_32 467 468 #define atomic_add_long atomic_add_64 469 #define atomic_clear_long atomic_clear_64 470 #define atomic_cmpset_long atomic_cmpset_64 471 #define atomic_fcmpset_long atomic_fcmpset_64 472 #define atomic_fetchadd_long atomic_fetchadd_64 473 #define atomic_readandclear_long atomic_readandclear_64 474 #define atomic_set_long atomic_set_64 475 #define atomic_subtract_long atomic_subtract_64 476 #define atomic_swap_long atomic_swap_64 477 478 #define atomic_add_ptr atomic_add_64 479 #define atomic_clear_ptr atomic_clear_64 480 #define atomic_cmpset_ptr atomic_cmpset_64 481 #define atomic_fcmpset_ptr atomic_fcmpset_64 482 #define atomic_fetchadd_ptr atomic_fetchadd_64 483 #define atomic_readandclear_ptr atomic_readandclear_64 484 #define atomic_set_ptr atomic_set_64 485 #define atomic_subtract_ptr atomic_subtract_64 486 #define atomic_swap_ptr atomic_swap_64 487 488 ATOMIC_ACQ_REL(set, 64) 489 ATOMIC_ACQ_REL(clear, 64) 490 ATOMIC_ACQ_REL(add, 64) 491 ATOMIC_ACQ_REL(subtract, 64) 492 493 ATOMIC_CMPSET_ACQ_REL(64); 494 ATOMIC_FCMPSET_ACQ_REL(64); 495 496 static __inline uint64_t 497 atomic_load_acq_64(volatile uint64_t *p) 498 { 499 uint64_t ret; 500 501 ret = *p; 502 503 fence(); 504 505 return (ret); 506 } 507 508 static __inline void 509 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 510 { 511 512 fence(); 513 514 *p = val; 515 } 516 517 #define atomic_add_acq_long atomic_add_acq_64 518 #define atomic_clear_acq_long atomic_clear_acq_64 519 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 520 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 521 #define atomic_load_acq_long atomic_load_acq_64 522 #define atomic_set_acq_long atomic_set_acq_64 523 #define atomic_subtract_acq_long atomic_subtract_acq_64 524 525 #define atomic_add_acq_ptr atomic_add_acq_64 526 #define atomic_clear_acq_ptr atomic_clear_acq_64 527 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 528 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 529 #define atomic_load_acq_ptr atomic_load_acq_64 530 #define atomic_set_acq_ptr atomic_set_acq_64 531 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 532 533 #undef ATOMIC_ACQ_REL 534 535 static __inline void 536 atomic_thread_fence_acq(void) 537 { 538 539 fence(); 540 } 541 542 static __inline void 543 atomic_thread_fence_rel(void) 544 { 545 546 fence(); 547 } 548 549 static __inline void 550 atomic_thread_fence_acq_rel(void) 551 { 552 553 fence(); 554 } 555 556 static __inline void 557 atomic_thread_fence_seq_cst(void) 558 { 559 560 fence(); 561 } 562 563 #define atomic_add_rel_long atomic_add_rel_64 564 #define atomic_clear_rel_long atomic_clear_rel_64 565 566 #define atomic_add_rel_long atomic_add_rel_64 567 #define atomic_clear_rel_long atomic_clear_rel_64 568 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 569 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 570 #define atomic_set_rel_long atomic_set_rel_64 571 #define atomic_subtract_rel_long atomic_subtract_rel_64 572 #define atomic_store_rel_long atomic_store_rel_64 573 574 #define atomic_add_rel_ptr atomic_add_rel_64 575 #define atomic_clear_rel_ptr atomic_clear_rel_64 576 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 577 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 578 #define atomic_set_rel_ptr atomic_set_rel_64 579 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 580 #define atomic_store_rel_ptr atomic_store_rel_64 581 582 #include <sys/_atomic_subword.h> 583 584 #endif /* _MACHINE_ATOMIC_H_ */ 585