1 /*- 2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD$ 35 */ 36 37 #ifndef _MACHINE_ATOMIC_H_ 38 #define _MACHINE_ATOMIC_H_ 39 40 #include <sys/atomic_common.h> 41 42 #define fence() __asm __volatile("fence" ::: "memory"); 43 #define mb() fence() 44 #define rmb() fence() 45 #define wmb() fence() 46 47 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); 48 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); 49 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); 50 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *, 51 uint16_t); 52 53 #define ATOMIC_ACQ_REL(NAME, WIDTH) \ 54 static __inline void \ 55 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 56 { \ 57 atomic_##NAME##_##WIDTH(p, v); \ 58 fence(); \ 59 } \ 60 \ 61 static __inline void \ 62 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 63 { \ 64 fence(); \ 65 atomic_##NAME##_##WIDTH(p, v); \ 66 } 67 68 #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \ 69 static __inline int \ 70 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 71 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 72 { \ 73 int retval; \ 74 \ 75 retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \ 76 fence(); \ 77 return (retval); \ 78 } \ 79 \ 80 static __inline int \ 81 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 82 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 83 { \ 84 fence(); \ 85 return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \ 86 } 87 88 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \ 89 static __inline int \ 90 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 91 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 92 { \ 93 int retval; \ 94 \ 95 retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \ 96 fence(); \ 97 return (retval); \ 98 } \ 99 \ 100 static __inline int \ 101 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 102 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 103 { \ 104 fence(); \ 105 return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \ 106 } 107 108 ATOMIC_CMPSET_ACQ_REL(8); 109 ATOMIC_FCMPSET_ACQ_REL(8); 110 ATOMIC_CMPSET_ACQ_REL(16); 111 ATOMIC_FCMPSET_ACQ_REL(16); 112 113 #define atomic_cmpset_char atomic_cmpset_8 114 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 115 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 116 #define atomic_fcmpset_char atomic_fcmpset_8 117 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 118 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 119 120 #define atomic_cmpset_short atomic_cmpset_16 121 #define atomic_cmpset_acq_short atomic_cmpset_acq_16 122 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 123 #define atomic_fcmpset_short atomic_fcmpset_16 124 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 125 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 126 127 static __inline void 128 atomic_add_32(volatile uint32_t *p, uint32_t val) 129 { 130 131 __asm __volatile("amoadd.w zero, %1, %0" 132 : "+A" (*p) 133 : "r" (val) 134 : "memory"); 135 } 136 137 static __inline void 138 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 139 { 140 141 __asm __volatile("amoadd.w zero, %1, %0" 142 : "+A" (*p) 143 : "r" (-val) 144 : "memory"); 145 } 146 147 static __inline void 148 atomic_set_32(volatile uint32_t *p, uint32_t val) 149 { 150 151 __asm __volatile("amoor.w zero, %1, %0" 152 : "+A" (*p) 153 : "r" (val) 154 : "memory"); 155 } 156 157 static __inline void 158 atomic_clear_32(volatile uint32_t *p, uint32_t val) 159 { 160 161 __asm __volatile("amoand.w zero, %1, %0" 162 : "+A" (*p) 163 : "r" (~val) 164 : "memory"); 165 } 166 167 static __inline int 168 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 169 { 170 uint32_t tmp; 171 int res; 172 173 res = 0; 174 175 __asm __volatile( 176 "0:" 177 "li %1, 1\n" /* Preset to fail */ 178 "lr.w %0, %2\n" 179 "bne %0, %z3, 1f\n" 180 "sc.w %1, %z4, %2\n" 181 "bnez %1, 0b\n" 182 "1:" 183 : "=&r" (tmp), "=&r" (res), "+A" (*p) 184 : "rJ" ((long)(int32_t)cmpval), "rJ" (newval) 185 : "memory"); 186 187 return (!res); 188 } 189 190 static __inline int 191 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 192 { 193 uint32_t tmp; 194 int res; 195 196 res = 0; 197 198 __asm __volatile( 199 "0:" 200 "li %1, 1\n" /* Preset to fail */ 201 "lr.w %0, %2\n" /* Load old value */ 202 "bne %0, %z4, 1f\n" /* Compare */ 203 "sc.w %1, %z5, %2\n" /* Try to store new value */ 204 "j 2f\n" 205 "1:" 206 "sw %0, %3\n" /* Save old value */ 207 "2:" 208 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 209 : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval) 210 : "memory"); 211 212 return (!res); 213 } 214 215 static __inline uint32_t 216 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 217 { 218 uint32_t ret; 219 220 __asm __volatile("amoadd.w %0, %2, %1" 221 : "=&r" (ret), "+A" (*p) 222 : "r" (val) 223 : "memory"); 224 225 return (ret); 226 } 227 228 static __inline uint32_t 229 atomic_readandclear_32(volatile uint32_t *p) 230 { 231 uint32_t ret; 232 uint32_t val; 233 234 val = 0; 235 236 __asm __volatile("amoswap.w %0, %2, %1" 237 : "=&r"(ret), "+A" (*p) 238 : "r" (val) 239 : "memory"); 240 241 return (ret); 242 } 243 244 #define atomic_add_int atomic_add_32 245 #define atomic_clear_int atomic_clear_32 246 #define atomic_cmpset_int atomic_cmpset_32 247 #define atomic_fcmpset_int atomic_fcmpset_32 248 #define atomic_fetchadd_int atomic_fetchadd_32 249 #define atomic_readandclear_int atomic_readandclear_32 250 #define atomic_set_int atomic_set_32 251 #define atomic_subtract_int atomic_subtract_32 252 253 ATOMIC_ACQ_REL(set, 32) 254 ATOMIC_ACQ_REL(clear, 32) 255 ATOMIC_ACQ_REL(add, 32) 256 ATOMIC_ACQ_REL(subtract, 32) 257 258 ATOMIC_CMPSET_ACQ_REL(32); 259 ATOMIC_FCMPSET_ACQ_REL(32); 260 261 static __inline uint32_t 262 atomic_load_acq_32(volatile uint32_t *p) 263 { 264 uint32_t ret; 265 266 ret = *p; 267 268 fence(); 269 270 return (ret); 271 } 272 273 static __inline void 274 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 275 { 276 277 fence(); 278 279 *p = val; 280 } 281 282 #define atomic_add_acq_int atomic_add_acq_32 283 #define atomic_clear_acq_int atomic_clear_acq_32 284 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 285 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 286 #define atomic_load_acq_int atomic_load_acq_32 287 #define atomic_set_acq_int atomic_set_acq_32 288 #define atomic_subtract_acq_int atomic_subtract_acq_32 289 290 #define atomic_add_rel_int atomic_add_rel_32 291 #define atomic_clear_rel_int atomic_clear_rel_32 292 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 293 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 294 #define atomic_set_rel_int atomic_set_rel_32 295 #define atomic_subtract_rel_int atomic_subtract_rel_32 296 #define atomic_store_rel_int atomic_store_rel_32 297 298 static __inline void 299 atomic_add_64(volatile uint64_t *p, uint64_t val) 300 { 301 302 __asm __volatile("amoadd.d zero, %1, %0" 303 : "+A" (*p) 304 : "r" (val) 305 : "memory"); 306 } 307 308 static __inline void 309 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 310 { 311 312 __asm __volatile("amoadd.d zero, %1, %0" 313 : "+A" (*p) 314 : "r" (-val) 315 : "memory"); 316 } 317 318 static __inline void 319 atomic_set_64(volatile uint64_t *p, uint64_t val) 320 { 321 322 __asm __volatile("amoor.d zero, %1, %0" 323 : "+A" (*p) 324 : "r" (val) 325 : "memory"); 326 } 327 328 static __inline void 329 atomic_clear_64(volatile uint64_t *p, uint64_t val) 330 { 331 332 __asm __volatile("amoand.d zero, %1, %0" 333 : "+A" (*p) 334 : "r" (~val) 335 : "memory"); 336 } 337 338 static __inline int 339 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 340 { 341 uint64_t tmp; 342 int res; 343 344 res = 0; 345 346 __asm __volatile( 347 "0:" 348 "li %1, 1\n" /* Preset to fail */ 349 "lr.d %0, %2\n" 350 "bne %0, %z3, 1f\n" 351 "sc.d %1, %z4, %2\n" 352 "bnez %1, 0b\n" 353 "1:" 354 : "=&r" (tmp), "=&r" (res), "+A" (*p) 355 : "rJ" (cmpval), "rJ" (newval) 356 : "memory"); 357 358 return (!res); 359 } 360 361 static __inline int 362 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 363 { 364 uint64_t tmp; 365 int res; 366 367 res = 0; 368 369 __asm __volatile( 370 "0:" 371 "li %1, 1\n" /* Preset to fail */ 372 "lr.d %0, %2\n" /* Load old value */ 373 "bne %0, %z4, 1f\n" /* Compare */ 374 "sc.d %1, %z5, %2\n" /* Try to store new value */ 375 "j 2f\n" 376 "1:" 377 "sd %0, %3\n" /* Save old value */ 378 "2:" 379 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 380 : "rJ" (*cmpval), "rJ" (newval) 381 : "memory"); 382 383 return (!res); 384 } 385 386 static __inline uint64_t 387 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 388 { 389 uint64_t ret; 390 391 __asm __volatile("amoadd.d %0, %2, %1" 392 : "=&r" (ret), "+A" (*p) 393 : "r" (val) 394 : "memory"); 395 396 return (ret); 397 } 398 399 static __inline uint64_t 400 atomic_readandclear_64(volatile uint64_t *p) 401 { 402 uint64_t ret; 403 uint64_t val; 404 405 val = 0; 406 407 __asm __volatile("amoswap.d %0, %2, %1" 408 : "=&r"(ret), "+A" (*p) 409 : "r" (val) 410 : "memory"); 411 412 return (ret); 413 } 414 415 static __inline uint32_t 416 atomic_swap_32(volatile uint32_t *p, uint32_t val) 417 { 418 uint32_t old; 419 420 __asm __volatile("amoswap.w %0, %2, %1" 421 : "=&r"(old), "+A" (*p) 422 : "r" (val) 423 : "memory"); 424 425 return (old); 426 } 427 428 static __inline uint64_t 429 atomic_swap_64(volatile uint64_t *p, uint64_t val) 430 { 431 uint64_t old; 432 433 __asm __volatile("amoswap.d %0, %2, %1" 434 : "=&r"(old), "+A" (*p) 435 : "r" (val) 436 : "memory"); 437 438 return (old); 439 } 440 441 #define atomic_swap_int atomic_swap_32 442 443 #define atomic_add_long atomic_add_64 444 #define atomic_clear_long atomic_clear_64 445 #define atomic_cmpset_long atomic_cmpset_64 446 #define atomic_fcmpset_long atomic_fcmpset_64 447 #define atomic_fetchadd_long atomic_fetchadd_64 448 #define atomic_readandclear_long atomic_readandclear_64 449 #define atomic_set_long atomic_set_64 450 #define atomic_subtract_long atomic_subtract_64 451 #define atomic_swap_long atomic_swap_64 452 453 #define atomic_add_ptr atomic_add_64 454 #define atomic_clear_ptr atomic_clear_64 455 #define atomic_cmpset_ptr atomic_cmpset_64 456 #define atomic_fcmpset_ptr atomic_fcmpset_64 457 #define atomic_fetchadd_ptr atomic_fetchadd_64 458 #define atomic_readandclear_ptr atomic_readandclear_64 459 #define atomic_set_ptr atomic_set_64 460 #define atomic_subtract_ptr atomic_subtract_64 461 #define atomic_swap_ptr atomic_swap_64 462 463 ATOMIC_ACQ_REL(set, 64) 464 ATOMIC_ACQ_REL(clear, 64) 465 ATOMIC_ACQ_REL(add, 64) 466 ATOMIC_ACQ_REL(subtract, 64) 467 468 ATOMIC_CMPSET_ACQ_REL(64); 469 ATOMIC_FCMPSET_ACQ_REL(64); 470 471 static __inline uint64_t 472 atomic_load_acq_64(volatile uint64_t *p) 473 { 474 uint64_t ret; 475 476 ret = *p; 477 478 fence(); 479 480 return (ret); 481 } 482 483 static __inline void 484 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 485 { 486 487 fence(); 488 489 *p = val; 490 } 491 492 #define atomic_add_acq_long atomic_add_acq_64 493 #define atomic_clear_acq_long atomic_clear_acq_64 494 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 495 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 496 #define atomic_load_acq_long atomic_load_acq_64 497 #define atomic_set_acq_long atomic_set_acq_64 498 #define atomic_subtract_acq_long atomic_subtract_acq_64 499 500 #define atomic_add_acq_ptr atomic_add_acq_64 501 #define atomic_clear_acq_ptr atomic_clear_acq_64 502 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 503 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 504 #define atomic_load_acq_ptr atomic_load_acq_64 505 #define atomic_set_acq_ptr atomic_set_acq_64 506 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 507 508 #undef ATOMIC_ACQ_REL 509 510 static __inline void 511 atomic_thread_fence_acq(void) 512 { 513 514 fence(); 515 } 516 517 static __inline void 518 atomic_thread_fence_rel(void) 519 { 520 521 fence(); 522 } 523 524 static __inline void 525 atomic_thread_fence_acq_rel(void) 526 { 527 528 fence(); 529 } 530 531 static __inline void 532 atomic_thread_fence_seq_cst(void) 533 { 534 535 fence(); 536 } 537 538 #define atomic_add_rel_long atomic_add_rel_64 539 #define atomic_clear_rel_long atomic_clear_rel_64 540 541 #define atomic_add_rel_long atomic_add_rel_64 542 #define atomic_clear_rel_long atomic_clear_rel_64 543 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 544 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 545 #define atomic_set_rel_long atomic_set_rel_64 546 #define atomic_subtract_rel_long atomic_subtract_rel_64 547 #define atomic_store_rel_long atomic_store_rel_64 548 549 #define atomic_add_rel_ptr atomic_add_rel_64 550 #define atomic_clear_rel_ptr atomic_clear_rel_64 551 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 552 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 553 #define atomic_set_rel_ptr atomic_set_rel_64 554 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 555 #define atomic_store_rel_ptr atomic_store_rel_64 556 557 #include <sys/_atomic_subword.h> 558 559 #endif /* _MACHINE_ATOMIC_H_ */ 560