1 /*- 2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #ifndef _MACHINE_ATOMIC_H_ 36 #define _MACHINE_ATOMIC_H_ 37 38 #include <sys/atomic_common.h> 39 40 #define fence() __asm __volatile("fence" ::: "memory"); 41 #define mb() fence() 42 #define rmb() fence() 43 #define wmb() fence() 44 45 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); 46 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); 47 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); 48 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *, 49 uint16_t); 50 51 #define ATOMIC_ACQ_REL(NAME, WIDTH) \ 52 static __inline void \ 53 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 54 { \ 55 atomic_##NAME##_##WIDTH(p, v); \ 56 fence(); \ 57 } \ 58 \ 59 static __inline void \ 60 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 61 { \ 62 fence(); \ 63 atomic_##NAME##_##WIDTH(p, v); \ 64 } 65 66 #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \ 67 static __inline int \ 68 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 69 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 70 { \ 71 int retval; \ 72 \ 73 retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \ 74 fence(); \ 75 return (retval); \ 76 } \ 77 \ 78 static __inline int \ 79 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 80 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ 81 { \ 82 fence(); \ 83 return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \ 84 } 85 86 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \ 87 static __inline int \ 88 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ 89 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 90 { \ 91 int retval; \ 92 \ 93 retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \ 94 fence(); \ 95 return (retval); \ 96 } \ 97 \ 98 static __inline int \ 99 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ 100 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ 101 { \ 102 fence(); \ 103 return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \ 104 } 105 106 ATOMIC_CMPSET_ACQ_REL(8); 107 ATOMIC_FCMPSET_ACQ_REL(8); 108 ATOMIC_CMPSET_ACQ_REL(16); 109 ATOMIC_FCMPSET_ACQ_REL(16); 110 111 #define atomic_cmpset_char atomic_cmpset_8 112 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 113 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 114 #define atomic_fcmpset_char atomic_fcmpset_8 115 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 116 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 117 118 #define atomic_cmpset_short atomic_cmpset_16 119 #define atomic_cmpset_acq_short atomic_cmpset_acq_16 120 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 121 #define atomic_fcmpset_short atomic_fcmpset_16 122 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 123 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 124 125 static __inline void 126 atomic_add_32(volatile uint32_t *p, uint32_t val) 127 { 128 129 __asm __volatile("amoadd.w zero, %1, %0" 130 : "+A" (*p) 131 : "r" (val) 132 : "memory"); 133 } 134 135 static __inline void 136 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 137 { 138 139 __asm __volatile("amoadd.w zero, %1, %0" 140 : "+A" (*p) 141 : "r" (-val) 142 : "memory"); 143 } 144 145 static __inline void 146 atomic_set_32(volatile uint32_t *p, uint32_t val) 147 { 148 149 __asm __volatile("amoor.w zero, %1, %0" 150 : "+A" (*p) 151 : "r" (val) 152 : "memory"); 153 } 154 155 static __inline void 156 atomic_clear_32(volatile uint32_t *p, uint32_t val) 157 { 158 159 __asm __volatile("amoand.w zero, %1, %0" 160 : "+A" (*p) 161 : "r" (~val) 162 : "memory"); 163 } 164 165 static __inline int 166 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 167 { 168 uint32_t tmp; 169 int res; 170 171 res = 0; 172 173 __asm __volatile( 174 "0:" 175 "li %1, 1\n" /* Preset to fail */ 176 "lr.w %0, %2\n" 177 "bne %0, %z3, 1f\n" 178 "sc.w %1, %z4, %2\n" 179 "bnez %1, 0b\n" 180 "1:" 181 : "=&r" (tmp), "=&r" (res), "+A" (*p) 182 : "rJ" ((long)(int32_t)cmpval), "rJ" (newval) 183 : "memory"); 184 185 return (!res); 186 } 187 188 static __inline int 189 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 190 { 191 uint32_t tmp; 192 int res; 193 194 res = 0; 195 196 __asm __volatile( 197 "0:" 198 "li %1, 1\n" /* Preset to fail */ 199 "lr.w %0, %2\n" /* Load old value */ 200 "bne %0, %z4, 1f\n" /* Compare */ 201 "sc.w %1, %z5, %2\n" /* Try to store new value */ 202 "j 2f\n" 203 "1:" 204 "sw %0, %3\n" /* Save old value */ 205 "2:" 206 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 207 : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval) 208 : "memory"); 209 210 return (!res); 211 } 212 213 static __inline uint32_t 214 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 215 { 216 uint32_t ret; 217 218 __asm __volatile("amoadd.w %0, %2, %1" 219 : "=&r" (ret), "+A" (*p) 220 : "r" (val) 221 : "memory"); 222 223 return (ret); 224 } 225 226 static __inline uint32_t 227 atomic_readandclear_32(volatile uint32_t *p) 228 { 229 uint32_t ret; 230 uint32_t val; 231 232 val = 0; 233 234 __asm __volatile("amoswap.w %0, %2, %1" 235 : "=&r"(ret), "+A" (*p) 236 : "r" (val) 237 : "memory"); 238 239 return (ret); 240 } 241 242 #define atomic_add_int atomic_add_32 243 #define atomic_clear_int atomic_clear_32 244 #define atomic_cmpset_int atomic_cmpset_32 245 #define atomic_fcmpset_int atomic_fcmpset_32 246 #define atomic_fetchadd_int atomic_fetchadd_32 247 #define atomic_readandclear_int atomic_readandclear_32 248 #define atomic_set_int atomic_set_32 249 #define atomic_subtract_int atomic_subtract_32 250 251 ATOMIC_ACQ_REL(set, 32) 252 ATOMIC_ACQ_REL(clear, 32) 253 ATOMIC_ACQ_REL(add, 32) 254 ATOMIC_ACQ_REL(subtract, 32) 255 256 ATOMIC_CMPSET_ACQ_REL(32); 257 ATOMIC_FCMPSET_ACQ_REL(32); 258 259 static __inline uint32_t 260 atomic_load_acq_32(volatile uint32_t *p) 261 { 262 uint32_t ret; 263 264 ret = *p; 265 266 fence(); 267 268 return (ret); 269 } 270 271 static __inline void 272 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 273 { 274 275 fence(); 276 277 *p = val; 278 } 279 280 #define atomic_add_acq_int atomic_add_acq_32 281 #define atomic_clear_acq_int atomic_clear_acq_32 282 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 283 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 284 #define atomic_load_acq_int atomic_load_acq_32 285 #define atomic_set_acq_int atomic_set_acq_32 286 #define atomic_subtract_acq_int atomic_subtract_acq_32 287 288 #define atomic_add_rel_int atomic_add_rel_32 289 #define atomic_clear_rel_int atomic_clear_rel_32 290 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 291 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 292 #define atomic_set_rel_int atomic_set_rel_32 293 #define atomic_subtract_rel_int atomic_subtract_rel_32 294 #define atomic_store_rel_int atomic_store_rel_32 295 296 static __inline void 297 atomic_add_64(volatile uint64_t *p, uint64_t val) 298 { 299 300 __asm __volatile("amoadd.d zero, %1, %0" 301 : "+A" (*p) 302 : "r" (val) 303 : "memory"); 304 } 305 306 static __inline void 307 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 308 { 309 310 __asm __volatile("amoadd.d zero, %1, %0" 311 : "+A" (*p) 312 : "r" (-val) 313 : "memory"); 314 } 315 316 static __inline void 317 atomic_set_64(volatile uint64_t *p, uint64_t val) 318 { 319 320 __asm __volatile("amoor.d zero, %1, %0" 321 : "+A" (*p) 322 : "r" (val) 323 : "memory"); 324 } 325 326 static __inline void 327 atomic_clear_64(volatile uint64_t *p, uint64_t val) 328 { 329 330 __asm __volatile("amoand.d zero, %1, %0" 331 : "+A" (*p) 332 : "r" (~val) 333 : "memory"); 334 } 335 336 static __inline int 337 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 338 { 339 uint64_t tmp; 340 int res; 341 342 res = 0; 343 344 __asm __volatile( 345 "0:" 346 "li %1, 1\n" /* Preset to fail */ 347 "lr.d %0, %2\n" 348 "bne %0, %z3, 1f\n" 349 "sc.d %1, %z4, %2\n" 350 "bnez %1, 0b\n" 351 "1:" 352 : "=&r" (tmp), "=&r" (res), "+A" (*p) 353 : "rJ" (cmpval), "rJ" (newval) 354 : "memory"); 355 356 return (!res); 357 } 358 359 static __inline int 360 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 361 { 362 uint64_t tmp; 363 int res; 364 365 res = 0; 366 367 __asm __volatile( 368 "0:" 369 "li %1, 1\n" /* Preset to fail */ 370 "lr.d %0, %2\n" /* Load old value */ 371 "bne %0, %z4, 1f\n" /* Compare */ 372 "sc.d %1, %z5, %2\n" /* Try to store new value */ 373 "j 2f\n" 374 "1:" 375 "sd %0, %3\n" /* Save old value */ 376 "2:" 377 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 378 : "rJ" (*cmpval), "rJ" (newval) 379 : "memory"); 380 381 return (!res); 382 } 383 384 static __inline uint64_t 385 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 386 { 387 uint64_t ret; 388 389 __asm __volatile("amoadd.d %0, %2, %1" 390 : "=&r" (ret), "+A" (*p) 391 : "r" (val) 392 : "memory"); 393 394 return (ret); 395 } 396 397 static __inline uint64_t 398 atomic_readandclear_64(volatile uint64_t *p) 399 { 400 uint64_t ret; 401 uint64_t val; 402 403 val = 0; 404 405 __asm __volatile("amoswap.d %0, %2, %1" 406 : "=&r"(ret), "+A" (*p) 407 : "r" (val) 408 : "memory"); 409 410 return (ret); 411 } 412 413 static __inline uint32_t 414 atomic_swap_32(volatile uint32_t *p, uint32_t val) 415 { 416 uint32_t old; 417 418 __asm __volatile("amoswap.w %0, %2, %1" 419 : "=&r"(old), "+A" (*p) 420 : "r" (val) 421 : "memory"); 422 423 return (old); 424 } 425 426 static __inline uint64_t 427 atomic_swap_64(volatile uint64_t *p, uint64_t val) 428 { 429 uint64_t old; 430 431 __asm __volatile("amoswap.d %0, %2, %1" 432 : "=&r"(old), "+A" (*p) 433 : "r" (val) 434 : "memory"); 435 436 return (old); 437 } 438 439 #define atomic_swap_int atomic_swap_32 440 441 #define atomic_add_long atomic_add_64 442 #define atomic_clear_long atomic_clear_64 443 #define atomic_cmpset_long atomic_cmpset_64 444 #define atomic_fcmpset_long atomic_fcmpset_64 445 #define atomic_fetchadd_long atomic_fetchadd_64 446 #define atomic_readandclear_long atomic_readandclear_64 447 #define atomic_set_long atomic_set_64 448 #define atomic_subtract_long atomic_subtract_64 449 #define atomic_swap_long atomic_swap_64 450 451 #define atomic_add_ptr atomic_add_64 452 #define atomic_clear_ptr atomic_clear_64 453 #define atomic_cmpset_ptr atomic_cmpset_64 454 #define atomic_fcmpset_ptr atomic_fcmpset_64 455 #define atomic_fetchadd_ptr atomic_fetchadd_64 456 #define atomic_readandclear_ptr atomic_readandclear_64 457 #define atomic_set_ptr atomic_set_64 458 #define atomic_subtract_ptr atomic_subtract_64 459 #define atomic_swap_ptr atomic_swap_64 460 461 ATOMIC_ACQ_REL(set, 64) 462 ATOMIC_ACQ_REL(clear, 64) 463 ATOMIC_ACQ_REL(add, 64) 464 ATOMIC_ACQ_REL(subtract, 64) 465 466 ATOMIC_CMPSET_ACQ_REL(64); 467 ATOMIC_FCMPSET_ACQ_REL(64); 468 469 static __inline uint64_t 470 atomic_load_acq_64(volatile uint64_t *p) 471 { 472 uint64_t ret; 473 474 ret = *p; 475 476 fence(); 477 478 return (ret); 479 } 480 481 static __inline void 482 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 483 { 484 485 fence(); 486 487 *p = val; 488 } 489 490 #define atomic_add_acq_long atomic_add_acq_64 491 #define atomic_clear_acq_long atomic_clear_acq_64 492 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 493 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 494 #define atomic_load_acq_long atomic_load_acq_64 495 #define atomic_set_acq_long atomic_set_acq_64 496 #define atomic_subtract_acq_long atomic_subtract_acq_64 497 498 #define atomic_add_acq_ptr atomic_add_acq_64 499 #define atomic_clear_acq_ptr atomic_clear_acq_64 500 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 501 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 502 #define atomic_load_acq_ptr atomic_load_acq_64 503 #define atomic_set_acq_ptr atomic_set_acq_64 504 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 505 506 #undef ATOMIC_ACQ_REL 507 508 static __inline void 509 atomic_thread_fence_acq(void) 510 { 511 512 fence(); 513 } 514 515 static __inline void 516 atomic_thread_fence_rel(void) 517 { 518 519 fence(); 520 } 521 522 static __inline void 523 atomic_thread_fence_acq_rel(void) 524 { 525 526 fence(); 527 } 528 529 static __inline void 530 atomic_thread_fence_seq_cst(void) 531 { 532 533 fence(); 534 } 535 536 #define atomic_add_rel_long atomic_add_rel_64 537 #define atomic_clear_rel_long atomic_clear_rel_64 538 539 #define atomic_add_rel_long atomic_add_rel_64 540 #define atomic_clear_rel_long atomic_clear_rel_64 541 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 542 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 543 #define atomic_set_rel_long atomic_set_rel_64 544 #define atomic_subtract_rel_long atomic_subtract_rel_64 545 #define atomic_store_rel_long atomic_store_rel_64 546 547 #define atomic_add_rel_ptr atomic_add_rel_64 548 #define atomic_clear_rel_ptr atomic_clear_rel_64 549 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 550 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 551 #define atomic_set_rel_ptr atomic_set_rel_64 552 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 553 #define atomic_store_rel_ptr atomic_store_rel_64 554 555 #include <sys/_atomic_subword.h> 556 557 #endif /* _MACHINE_ATOMIC_H_ */ 558