1 /*- 2 * Copyright (c) 2008 Marcel Moolenaar 3 * Copyright (c) 2001 Benno Rice 4 * Copyright (c) 2001 David E. O'Brien 5 * Copyright (c) 1998 Doug Rabson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #ifndef _MACHINE_ATOMIC_H_ 33 #define _MACHINE_ATOMIC_H_ 34 35 #ifndef _SYS_CDEFS_H_ 36 #error this file needs sys/cdefs.h as a prerequisite 37 #endif 38 39 #define __ATOMIC_BARRIER \ 40 __asm __volatile("sync" : : : "memory") 41 42 /* 43 * atomic_add(p, v) 44 * { *p += v; } 45 */ 46 47 #define __ATOMIC_ADD_8(p, v, t) \ 48 8-bit atomic_add not implemented 49 50 #define __ATOMIC_ADD_16(p, v, t) \ 51 16-bit atomic_add not implemented 52 53 #define __ATOMIC_ADD_32(p, v, t) \ 54 __asm __volatile( \ 55 "1: lwarx %0, 0, %2\n" \ 56 " add %0, %3, %0\n" \ 57 " stwcx. %0, 0, %2\n" \ 58 " bne- 1b\n" \ 59 : "=&r" (t), "=m" (*p) \ 60 : "r" (p), "r" (v), "m" (*p) \ 61 : "cc", "memory") \ 62 /* __ATOMIC_ADD_32 */ 63 64 #define __ATOMIC_ADD_64(p, v, t) \ 65 64-bit atomic_add not implemented 66 67 #define _ATOMIC_ADD(width, suffix, type) \ 68 static __inline void \ 69 atomic_add_##suffix(volatile type *p, type v) { \ 70 type t; \ 71 __ATOMIC_ADD_##width(p, v, t); \ 72 } \ 73 \ 74 static __inline void \ 75 atomic_add_acq_##suffix(volatile type *p, type v) { \ 76 type t; \ 77 __ATOMIC_ADD_##width(p, v, t); \ 78 __ATOMIC_BARRIER; \ 79 } \ 80 \ 81 static __inline void \ 82 atomic_add_rel_##suffix(volatile type *p, type v) { \ 83 type t; \ 84 __ATOMIC_BARRIER; \ 85 __ATOMIC_ADD_##width(p, v, t); \ 86 } \ 87 /* _ATOMIC_ADD */ 88 89 #if 0 90 _ATOMIC_ADD(8, 8, uint8_t) 91 _ATOMIC_ADD(8, char, u_char) 92 _ATOMIC_ADD(16, 16, uint16_t) 93 _ATOMIC_ADD(16, short, u_short) 94 #endif 95 _ATOMIC_ADD(32, 32, uint32_t) 96 _ATOMIC_ADD(32, int, u_int) 97 _ATOMIC_ADD(32, long, u_long) 98 _ATOMIC_ADD(32, ptr, uintptr_t) 99 #if 0 100 _ATOMIC_ADD(64, 64, uint64_t) 101 _ATOMIC_ADD(64, long_long, u_long_long) 102 #endif 103 104 #undef _ATOMIC_ADD 105 #undef __ATOMIC_ADD_64 106 #undef __ATOMIC_ADD_32 107 #undef __ATOMIC_ADD_16 108 #undef __ATOMIC_ADD_8 109 110 /* 111 * atomic_clear(p, v) 112 * { *p &= ~v; } 113 */ 114 115 #define __ATOMIC_CLEAR_8(p, v, t) \ 116 8-bit atomic_clear not implemented 117 118 #define __ATOMIC_CLEAR_16(p, v, t) \ 119 16-bit atomic_clear not implemented 120 121 #define __ATOMIC_CLEAR_32(p, v, t) \ 122 __asm __volatile( \ 123 "1: lwarx %0, 0, %2\n" \ 124 " andc %0, %0, %3\n" \ 125 " stwcx. %0, 0, %2\n" \ 126 " bne- 1b\n" \ 127 : "=&r" (t), "=m" (*p) \ 128 : "r" (p), "r" (v), "m" (*p) \ 129 : "cc", "memory") \ 130 /* __ATOMIC_CLEAR_32 */ 131 132 #define __ATOMIC_CLEAR_64(p, v, t) \ 133 64-bit atomic_clear not implemented 134 135 #define _ATOMIC_CLEAR(width, suffix, type) \ 136 static __inline void \ 137 atomic_clear_##suffix(volatile type *p, type v) { \ 138 type t; \ 139 __ATOMIC_CLEAR_##width(p, v, t); \ 140 } \ 141 \ 142 static __inline void \ 143 atomic_clear_acq_##suffix(volatile type *p, type v) { \ 144 type t; \ 145 __ATOMIC_CLEAR_##width(p, v, t); \ 146 __ATOMIC_BARRIER; \ 147 } \ 148 \ 149 static __inline void \ 150 atomic_clear_rel_##suffix(volatile type *p, type v) { \ 151 type t; \ 152 __ATOMIC_BARRIER; \ 153 __ATOMIC_CLEAR_##width(p, v, t); \ 154 } \ 155 /* _ATOMIC_CLEAR */ 156 157 #if 0 158 _ATOMIC_CLEAR(8, 8, uint8_t) 159 _ATOMIC_CLEAR(8, char, u_char) 160 _ATOMIC_CLEAR(16, 16, uint16_t) 161 _ATOMIC_CLEAR(16, short, u_short) 162 #endif 163 _ATOMIC_CLEAR(32, 32, uint32_t) 164 _ATOMIC_CLEAR(32, int, u_int) 165 _ATOMIC_CLEAR(32, long, u_long) 166 _ATOMIC_CLEAR(32, ptr, uintptr_t) 167 #if 0 168 _ATOMIC_CLEAR(64, 64, uint64_t) 169 _ATOMIC_CLEAR(64, long_long, u_long_long) 170 #endif 171 172 #undef _ATOMIC_CLEAR 173 #undef __ATOMIC_CLEAR_64 174 #undef __ATOMIC_CLEAR_32 175 #undef __ATOMIC_CLEAR_16 176 #undef __ATOMIC_CLEAR_8 177 178 /* 179 * atomic_cmpset(p, o, n) 180 */ 181 /* TODO -- see below */ 182 183 /* 184 * atomic_load_acq(p) 185 */ 186 /* TODO -- see below */ 187 188 /* 189 * atomic_readandclear(p) 190 */ 191 /* TODO -- see below */ 192 193 /* 194 * atomic_set(p, v) 195 * { *p |= v; } 196 */ 197 198 #define __ATOMIC_SET_8(p, v, t) \ 199 8-bit atomic_set not implemented 200 201 #define __ATOMIC_SET_16(p, v, t) \ 202 16-bit atomic_set not implemented 203 204 #define __ATOMIC_SET_32(p, v, t) \ 205 __asm __volatile( \ 206 "1: lwarx %0, 0, %2\n" \ 207 " or %0, %3, %0\n" \ 208 " stwcx. %0, 0, %2\n" \ 209 " bne- 1b\n" \ 210 : "=&r" (t), "=m" (*p) \ 211 : "r" (p), "r" (v), "m" (*p) \ 212 : "cc", "memory") \ 213 /* __ATOMIC_SET_32 */ 214 215 #define __ATOMIC_SET_64(p, v, t) \ 216 64-bit atomic_set not implemented 217 218 #define _ATOMIC_SET(width, suffix, type) \ 219 static __inline void \ 220 atomic_set_##suffix(volatile type *p, type v) { \ 221 type t; \ 222 __ATOMIC_SET_##width(p, v, t); \ 223 } \ 224 \ 225 static __inline void \ 226 atomic_set_acq_##suffix(volatile type *p, type v) { \ 227 type t; \ 228 __ATOMIC_SET_##width(p, v, t); \ 229 __ATOMIC_BARRIER; \ 230 } \ 231 \ 232 static __inline void \ 233 atomic_set_rel_##suffix(volatile type *p, type v) { \ 234 type t; \ 235 __ATOMIC_BARRIER; \ 236 __ATOMIC_SET_##width(p, v, t); \ 237 } \ 238 /* _ATOMIC_SET */ 239 240 #if 0 241 _ATOMIC_SET(8, 8, uint8_t) 242 _ATOMIC_SET(8, char, u_char) 243 _ATOMIC_SET(16, 16, uint16_t) 244 _ATOMIC_SET(16, short, u_short) 245 #endif 246 _ATOMIC_SET(32, 32, uint32_t) 247 _ATOMIC_SET(32, int, u_int) 248 _ATOMIC_SET(32, long, u_long) 249 _ATOMIC_SET(32, ptr, uintptr_t) 250 #if 0 251 _ATOMIC_SET(64, 64, uint64_t) 252 _ATOMIC_SET(64, long_long, u_long_long) 253 #endif 254 255 #undef _ATOMIC_SET 256 #undef __ATOMIC_SET_64 257 #undef __ATOMIC_SET_32 258 #undef __ATOMIC_SET_16 259 #undef __ATOMIC_SET_8 260 261 /* 262 * atomic_subtract(p, v) 263 * { *p -= v; } 264 */ 265 266 #define __ATOMIC_SUBTRACT_8(p, v, t) \ 267 8-bit atomic_subtract not implemented 268 269 #define __ATOMIC_SUBTRACT_16(p, v, t) \ 270 16-bit atomic_subtract not implemented 271 272 #define __ATOMIC_SUBTRACT_32(p, v, t) \ 273 __asm __volatile( \ 274 "1: lwarx %0, 0, %2\n" \ 275 " subf %0, %3, %0\n" \ 276 " stwcx. %0, 0, %2\n" \ 277 " bne- 1b\n" \ 278 : "=&r" (t), "=m" (*p) \ 279 : "r" (p), "r" (v), "m" (*p) \ 280 : "cc", "memory") \ 281 /* __ATOMIC_SUBTRACT_32 */ 282 283 #define __ATOMIC_SUBTRACT_64(p, v, t) \ 284 64-bit atomic_subtract not implemented 285 286 #define _ATOMIC_SUBTRACT(width, suffix, type) \ 287 static __inline void \ 288 atomic_subtract_##suffix(volatile type *p, type v) { \ 289 type t; \ 290 __ATOMIC_SUBTRACT_##width(p, v, t); \ 291 } \ 292 \ 293 static __inline void \ 294 atomic_subtract_acq_##suffix(volatile type *p, type v) { \ 295 type t; \ 296 __ATOMIC_SUBTRACT_##width(p, v, t); \ 297 __ATOMIC_BARRIER; \ 298 } \ 299 \ 300 static __inline void \ 301 atomic_subtract_rel_##suffix(volatile type *p, type v) { \ 302 type t; \ 303 __ATOMIC_BARRIER; \ 304 __ATOMIC_SUBTRACT_##width(p, v, t); \ 305 } \ 306 /* _ATOMIC_SUBTRACT */ 307 308 #if 0 309 _ATOMIC_SUBTRACT(8, 8, uint8_t) 310 _ATOMIC_SUBTRACT(8, char, u_char) 311 _ATOMIC_SUBTRACT(16, 16, uint16_t) 312 _ATOMIC_SUBTRACT(16, short, u_short) 313 #endif 314 _ATOMIC_SUBTRACT(32, 32, uint32_t) 315 _ATOMIC_SUBTRACT(32, int, u_int) 316 _ATOMIC_SUBTRACT(32, long, u_long) 317 _ATOMIC_SUBTRACT(32, ptr, uintptr_t) 318 #if 0 319 _ATOMIC_SUBTRACT(64, 64, uint64_t) 320 _ATOMIC_SUBTRACT(64, long_long, u_long_long) 321 #endif 322 323 #undef _ATOMIC_SUBTRACT 324 #undef __ATOMIC_SUBTRACT_64 325 #undef __ATOMIC_SUBTRACT_32 326 #undef __ATOMIC_SUBTRACT_16 327 #undef __ATOMIC_SUBTRACT_8 328 329 /* 330 * atomic_store_rel(p, v) 331 */ 332 /* TODO -- see below */ 333 334 /* 335 * Old/original implementations that still need revisiting. 336 */ 337 338 static __inline uint32_t 339 atomic_readandclear_32(volatile uint32_t *addr) 340 { 341 uint32_t result,temp; 342 343 #ifdef __GNUCLIKE_ASM 344 __asm __volatile ( 345 "\tsync\n" /* drain writes */ 346 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 347 "li %1, 0\n\t" /* load new value */ 348 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 349 "bne- 1b\n\t" /* spin if failed */ 350 : "=&r"(result), "=&r"(temp), "=m" (*addr) 351 : "r" (addr), "m" (*addr) 352 : "cc", "memory"); 353 #endif 354 355 return (result); 356 } 357 358 #define atomic_readandclear_int atomic_readandclear_32 359 #define atomic_readandclear_long atomic_readandclear_32 360 #define atomic_readandclear_ptr atomic_readandclear_32 361 362 /* 363 * We assume that a = b will do atomic loads and stores. 364 */ 365 #define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ 366 static __inline u_##TYPE \ 367 atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ 368 { \ 369 u_##TYPE v; \ 370 \ 371 v = *p; \ 372 __ATOMIC_BARRIER; \ 373 return (v); \ 374 } \ 375 \ 376 static __inline void \ 377 atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ 378 { \ 379 __ATOMIC_BARRIER; \ 380 *p = v; \ 381 } \ 382 \ 383 static __inline u_##TYPE \ 384 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 385 { \ 386 u_##TYPE v; \ 387 \ 388 v = *p; \ 389 __ATOMIC_BARRIER; \ 390 return (v); \ 391 } \ 392 \ 393 static __inline void \ 394 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 395 { \ 396 __ATOMIC_BARRIER; \ 397 *p = v; \ 398 } 399 400 ATOMIC_STORE_LOAD(char, 8) 401 ATOMIC_STORE_LOAD(short, 16) 402 ATOMIC_STORE_LOAD(int, 32) 403 404 #define atomic_load_acq_long atomic_load_acq_32 405 #define atomic_store_rel_long atomic_store_rel_32 406 #define atomic_load_acq_ptr atomic_load_acq_32 407 #define atomic_store_rel_ptr atomic_store_rel_32 408 409 #undef ATOMIC_STORE_LOAD 410 411 /* 412 * Atomically compare the value stored at *p with cmpval and if the 413 * two values are equal, update the value of *p with newval. Returns 414 * zero if the compare failed, nonzero otherwise. 415 */ 416 static __inline uint32_t 417 atomic_cmpset_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) 418 { 419 uint32_t ret; 420 421 #ifdef __GNUCLIKE_ASM 422 __asm __volatile ( 423 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 424 "cmplw %3, %0\n\t" /* compare */ 425 "bne 2f\n\t" /* exit if not equal */ 426 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 427 "bne- 1b\n\t" /* spin if failed */ 428 "li %0, 1\n\t" /* success - retval = 1 */ 429 "b 3f\n\t" /* we've succeeded */ 430 "2:\n\t" 431 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 432 "li %0, 0\n\t" /* failure - retval = 0 */ 433 "3:\n\t" 434 : "=&r" (ret), "=m" (*p) 435 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 436 : "cc", "memory"); 437 #endif 438 439 return (ret); 440 } 441 442 static __inline u_long 443 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 444 { 445 uint32_t ret; 446 447 #ifdef __GNUCLIKE_ASM 448 __asm __volatile ( 449 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 450 "cmplw %3, %0\n\t" /* compare */ 451 "bne 2f\n\t" /* exit if not equal */ 452 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 453 "bne- 1b\n\t" /* spin if failed */ 454 "li %0, 1\n\t" /* success - retval = 1 */ 455 "b 3f\n\t" /* we've succeeded */ 456 "2:\n\t" 457 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 458 "li %0, 0\n\t" /* failure - retval = 0 */ 459 "3:\n\t" 460 : "=&r" (ret), "=m" (*p) 461 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 462 : "cc", "memory"); 463 #endif 464 465 return (ret); 466 } 467 468 #define atomic_cmpset_int atomic_cmpset_32 469 470 #define atomic_cmpset_ptr(dst, old, new) \ 471 atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 472 473 static __inline uint32_t 474 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 475 { 476 int retval; 477 478 retval = atomic_cmpset_32(p, cmpval, newval); 479 __ATOMIC_BARRIER; 480 return (retval); 481 } 482 483 static __inline uint32_t 484 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 485 { 486 __ATOMIC_BARRIER; 487 return (atomic_cmpset_32(p, cmpval, newval)); 488 } 489 490 static __inline u_long 491 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) 492 { 493 int retval; 494 495 retval = atomic_cmpset_long(p, cmpval, newval); 496 __ATOMIC_BARRIER; 497 return (retval); 498 } 499 500 static __inline uint32_t 501 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) 502 { 503 __ATOMIC_BARRIER; 504 return (atomic_cmpset_long(p, cmpval, newval)); 505 } 506 507 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 508 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 509 510 #define atomic_cmpset_acq_ptr(dst, old, new) \ 511 atomic_cmpset_acq_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 512 #define atomic_cmpset_rel_ptr(dst, old, new) \ 513 atomic_cmpset_rel_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 514 515 static __inline uint32_t 516 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 517 { 518 uint32_t value; 519 520 do { 521 value = *p; 522 } while (!atomic_cmpset_32(p, value, value + v)); 523 return (value); 524 } 525 526 #define atomic_fetchadd_int atomic_fetchadd_32 527 #define atomic_fetchadd_long(p, v) \ 528 (u_long)atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v)) 529 530 #undef __ATOMIC_BARRIER 531 532 #endif /* ! _MACHINE_ATOMIC_H_ */ 533