1 /*- 2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD$ 35 */ 36 37 #ifndef _MACHINE_ATOMIC_H_ 38 #define _MACHINE_ATOMIC_H_ 39 40 #define fence() __asm __volatile("fence" ::: "memory"); 41 #define mb() fence() 42 #define rmb() fence() 43 #define wmb() fence() 44 45 #define ATOMIC_ACQ_REL(NAME, WIDTH) \ 46 static __inline void \ 47 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 48 { \ 49 atomic_##NAME##_##WIDTH(p, v); \ 50 fence(); \ 51 } \ 52 \ 53 static __inline void \ 54 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 55 { \ 56 fence(); \ 57 atomic_##NAME##_##WIDTH(p, v); \ 58 } 59 60 static __inline void 61 atomic_add_32(volatile uint32_t *p, uint32_t val) 62 { 63 64 __asm __volatile("amoadd.w zero, %1, %0" 65 : "+A" (*p) 66 : "r" (val) 67 : "memory"); 68 } 69 70 static __inline void 71 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 72 { 73 74 __asm __volatile("amoadd.w zero, %1, %0" 75 : "+A" (*p) 76 : "r" (-val) 77 : "memory"); 78 } 79 80 static __inline void 81 atomic_set_32(volatile uint32_t *p, uint32_t val) 82 { 83 84 __asm __volatile("amoor.w zero, %1, %0" 85 : "+A" (*p) 86 : "r" (val) 87 : "memory"); 88 } 89 90 static __inline void 91 atomic_clear_32(volatile uint32_t *p, uint32_t val) 92 { 93 94 __asm __volatile("amoand.w zero, %1, %0" 95 : "+A" (*p) 96 : "r" (~val) 97 : "memory"); 98 } 99 100 static __inline int 101 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 102 { 103 uint32_t tmp; 104 int res; 105 106 res = 0; 107 108 __asm __volatile( 109 "0:" 110 "li %1, 1\n" /* Preset to fail */ 111 "lr.w %0, %2\n" 112 "bne %0, %z3, 1f\n" 113 "sc.w %1, %z4, %2\n" 114 "bnez %1, 0b\n" 115 "1:" 116 : "=&r" (tmp), "=&r" (res), "+A" (*p) 117 : "rJ" (cmpval), "rJ" (newval) 118 : "memory"); 119 120 return (!res); 121 } 122 123 static __inline uint32_t 124 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 125 { 126 uint32_t ret; 127 128 __asm __volatile("amoadd.w %0, %2, %1" 129 : "=&r" (ret), "+A" (*p) 130 : "r" (val) 131 : "memory"); 132 133 return (ret); 134 } 135 136 static __inline uint32_t 137 atomic_readandclear_32(volatile uint32_t *p) 138 { 139 uint32_t ret; 140 uint32_t val; 141 142 val = 0; 143 144 __asm __volatile("amoswap.w %0, %2, %1" 145 : "=&r"(ret), "+A" (*p) 146 : "r" (val) 147 : "memory"); 148 149 return (ret); 150 } 151 152 #define atomic_add_int atomic_add_32 153 #define atomic_clear_int atomic_clear_32 154 #define atomic_cmpset_int atomic_cmpset_32 155 #define atomic_fetchadd_int atomic_fetchadd_32 156 #define atomic_readandclear_int atomic_readandclear_32 157 #define atomic_set_int atomic_set_32 158 #define atomic_subtract_int atomic_subtract_32 159 160 ATOMIC_ACQ_REL(set, 32) 161 ATOMIC_ACQ_REL(clear, 32) 162 ATOMIC_ACQ_REL(add, 32) 163 ATOMIC_ACQ_REL(subtract, 32) 164 165 static __inline int 166 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 167 { 168 int res; 169 170 res = atomic_cmpset_32(p, cmpval, newval); 171 172 fence(); 173 174 return (res); 175 } 176 177 static __inline int 178 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 179 { 180 181 fence(); 182 183 return (atomic_cmpset_32(p, cmpval, newval)); 184 } 185 186 static __inline uint32_t 187 atomic_load_acq_32(volatile uint32_t *p) 188 { 189 uint32_t ret; 190 191 ret = *p; 192 193 fence(); 194 195 return (ret); 196 } 197 198 static __inline void 199 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 200 { 201 202 fence(); 203 204 *p = val; 205 } 206 207 #define atomic_add_acq_int atomic_add_acq_32 208 #define atomic_clear_acq_int atomic_clear_acq_32 209 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 210 #define atomic_load_acq_int atomic_load_acq_32 211 #define atomic_set_acq_int atomic_set_acq_32 212 #define atomic_subtract_acq_int atomic_subtract_acq_32 213 214 #define atomic_add_rel_int atomic_add_rel_32 215 #define atomic_clear_rel_int atomic_add_rel_32 216 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 217 #define atomic_set_rel_int atomic_set_rel_32 218 #define atomic_subtract_rel_int atomic_subtract_rel_32 219 #define atomic_store_rel_int atomic_store_rel_32 220 221 static __inline void 222 atomic_add_64(volatile uint64_t *p, uint64_t val) 223 { 224 225 __asm __volatile("amoadd.d zero, %1, %0" 226 : "+A" (*p) 227 : "r" (val) 228 : "memory"); 229 } 230 231 static __inline void 232 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 233 { 234 235 __asm __volatile("amoadd.d zero, %1, %0" 236 : "+A" (*p) 237 : "r" (-val) 238 : "memory"); 239 } 240 241 static __inline void 242 atomic_set_64(volatile uint64_t *p, uint64_t val) 243 { 244 245 __asm __volatile("amoor.d zero, %1, %0" 246 : "+A" (*p) 247 : "r" (val) 248 : "memory"); 249 } 250 251 static __inline void 252 atomic_clear_64(volatile uint64_t *p, uint64_t val) 253 { 254 255 __asm __volatile("amoand.d zero, %1, %0" 256 : "+A" (*p) 257 : "r" (~val) 258 : "memory"); 259 } 260 261 static __inline int 262 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 263 { 264 uint64_t tmp; 265 int res; 266 267 res = 0; 268 269 __asm __volatile( 270 "0:" 271 "li %1, 1\n" /* Preset to fail */ 272 "lr.d %0, %2\n" 273 "bne %0, %z3, 1f\n" 274 "sc.d %1, %z4, %2\n" 275 "bnez %1, 0b\n" 276 "1:" 277 : "=&r" (tmp), "=&r" (res), "+A" (*p) 278 : "rJ" (cmpval), "rJ" (newval) 279 : "memory"); 280 281 return (!res); 282 } 283 284 static __inline uint64_t 285 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 286 { 287 uint64_t ret; 288 289 __asm __volatile("amoadd.d %0, %2, %1" 290 : "=&r" (ret), "+A" (*p) 291 : "r" (val) 292 : "memory"); 293 294 return (ret); 295 } 296 297 static __inline uint64_t 298 atomic_readandclear_64(volatile uint64_t *p) 299 { 300 uint64_t ret; 301 uint64_t val; 302 303 val = 0; 304 305 __asm __volatile("amoswap.d %0, %2, %1" 306 : "=&r"(ret), "+A" (*p) 307 : "r" (val) 308 : "memory"); 309 310 return (ret); 311 } 312 313 static __inline uint64_t 314 atomic_swap_64(volatile uint64_t *p, uint64_t val) 315 { 316 uint64_t old; 317 318 __asm __volatile("amoswap.d %0, %2, %1" 319 : "=&r"(old), "+A" (*p) 320 : "r" (val) 321 : "memory"); 322 323 return (old); 324 } 325 326 #define atomic_add_long atomic_add_64 327 #define atomic_clear_long atomic_clear_64 328 #define atomic_cmpset_long atomic_cmpset_64 329 #define atomic_fetchadd_long atomic_fetchadd_64 330 #define atomic_readandclear_long atomic_readandclear_64 331 #define atomic_set_long atomic_set_64 332 #define atomic_subtract_long atomic_subtract_64 333 334 #define atomic_add_ptr atomic_add_64 335 #define atomic_clear_ptr atomic_clear_64 336 #define atomic_cmpset_ptr atomic_cmpset_64 337 #define atomic_fetchadd_ptr atomic_fetchadd_64 338 #define atomic_readandclear_ptr atomic_readandclear_64 339 #define atomic_set_ptr atomic_set_64 340 #define atomic_subtract_ptr atomic_subtract_64 341 342 ATOMIC_ACQ_REL(set, 64) 343 ATOMIC_ACQ_REL(clear, 64) 344 ATOMIC_ACQ_REL(add, 64) 345 ATOMIC_ACQ_REL(subtract, 64) 346 347 static __inline int 348 atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 349 { 350 int res; 351 352 res = atomic_cmpset_64(p, cmpval, newval); 353 354 fence(); 355 356 return (res); 357 } 358 359 static __inline int 360 atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 361 { 362 363 fence(); 364 365 return (atomic_cmpset_64(p, cmpval, newval)); 366 } 367 368 static __inline uint64_t 369 atomic_load_acq_64(volatile uint64_t *p) 370 { 371 uint64_t ret; 372 373 ret = *p; 374 375 fence(); 376 377 return (ret); 378 } 379 380 static __inline void 381 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 382 { 383 384 fence(); 385 386 *p = val; 387 } 388 389 #define atomic_add_acq_int atomic_add_acq_32 390 #define atomic_clear_acq_int atomic_clear_acq_32 391 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 392 393 #define atomic_add_acq_long atomic_add_acq_64 394 #define atomic_clear_acq_long atomic_add_acq_64 395 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 396 #define atomic_load_acq_long atomic_load_acq_64 397 #define atomic_set_acq_long atomic_set_acq_64 398 #define atomic_subtract_acq_long atomic_subtract_acq_64 399 400 #define atomic_add_acq_ptr atomic_add_acq_64 401 #define atomic_clear_acq_ptr atomic_add_acq_64 402 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 403 #define atomic_load_acq_ptr atomic_load_acq_64 404 #define atomic_set_acq_ptr atomic_set_acq_64 405 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 406 407 static __inline void 408 atomic_thread_fence_acq(void) 409 { 410 411 fence(); 412 } 413 414 static __inline void 415 atomic_thread_fence_rel(void) 416 { 417 418 fence(); 419 } 420 421 static __inline void 422 atomic_thread_fence_acq_rel(void) 423 { 424 425 fence(); 426 } 427 428 static __inline void 429 atomic_thread_fence_seq_cst(void) 430 { 431 432 fence(); 433 } 434 435 #define atomic_add_rel_long atomic_add_rel_64 436 #define atomic_clear_rel_long atomic_clear_rel_64 437 438 #define atomic_add_rel_long atomic_add_rel_64 439 #define atomic_clear_rel_long atomic_clear_rel_64 440 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 441 #define atomic_set_rel_long atomic_set_rel_64 442 #define atomic_subtract_rel_long atomic_subtract_rel_64 443 #define atomic_store_rel_long atomic_store_rel_64 444 445 #define atomic_add_rel_ptr atomic_add_rel_64 446 #define atomic_clear_rel_ptr atomic_clear_rel_64 447 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 448 #define atomic_set_rel_ptr atomic_set_rel_64 449 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 450 #define atomic_store_rel_ptr atomic_store_rel_64 451 452 #endif /* _MACHINE_ATOMIC_H_ */ 453