1 /*- 2 * Copyright (c) 2001 Benno Rice 3 * Copyright (c) 2001 David E. O'Brien 4 * Copyright (c) 1998 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #ifndef _MACHINE_ATOMIC_H_ 32 #define _MACHINE_ATOMIC_H_ 33 34 #include <machine/cpufunc.h> 35 36 /* 37 * Various simple arithmetic on memory which is atomic in the presence 38 * of interrupts and SMP safe. 39 */ 40 41 void atomic_set_8(volatile uint8_t *, uint8_t); 42 void atomic_clear_8(volatile uint8_t *, uint8_t); 43 void atomic_add_8(volatile uint8_t *, uint8_t); 44 void atomic_subtract_8(volatile uint8_t *, uint8_t); 45 46 void atomic_set_16(volatile uint16_t *, uint16_t); 47 void atomic_clear_16(volatile uint16_t *, uint16_t); 48 void atomic_add_16(volatile uint16_t *, uint16_t); 49 void atomic_subtract_16(volatile uint16_t *, uint16_t); 50 51 static __inline void 52 atomic_set_32(volatile uint32_t *p, uint32_t v) 53 { 54 uint32_t temp; 55 56 #ifdef __GNUC__ 57 __asm __volatile ( 58 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 59 "or %0, %3, %0\n\t" /* calculate new value */ 60 "stwcx. %0, 0, %2\n\t" /* attempt to store */ 61 "bne- 1b\n\t" /* spin if failed */ 62 : "=&r" (temp), "+m" (*p) 63 : "r" (p), "r" (v) 64 : "cc", "memory"); 65 #endif 66 } 67 68 static __inline void 69 atomic_clear_32(volatile uint32_t *p, uint32_t v) 70 { 71 uint32_t temp; 72 73 #ifdef __GNUC__ 74 __asm __volatile ( 75 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 76 "andc %0, %0, %3\n\t" /* calculate new value */ 77 "stwcx. %0, 0, %2\n\t" /* attempt to store */ 78 "bne- 1b\n\t" /* spin if failed */ 79 : "=&r" (temp), "+m" (*p) 80 : "r" (p), "r" (v) 81 : "cc", "memory"); 82 #endif 83 } 84 85 static __inline void 86 atomic_add_32(volatile uint32_t *p, uint32_t v) 87 { 88 uint32_t temp; 89 90 #ifdef __GNUC__ 91 __asm __volatile ( 92 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 93 "add %0, %3, %0\n\t" /* calculate new value */ 94 "stwcx. %0, 0, %2\n\t" /* attempt to store */ 95 "bne- 1b\n\t" /* spin if failed */ 96 : "=&r" (temp), "+m" (*p) 97 : "r" (p), "r" (v) 98 : "cc", "memory"); 99 #endif 100 } 101 102 static __inline void 103 atomic_subtract_32(volatile uint32_t *p, uint32_t v) 104 { 105 uint32_t temp; 106 107 #ifdef __GNUC__ 108 __asm __volatile ( 109 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 110 "subf %0, %3, %0\n\t" /* calculate new value */ 111 "stwcx. %0, 0, %2\n\t" /* attempt to store */ 112 "bne- 1b\n\t" /* spin if failed */ 113 : "=&r" (temp), "+m" (*p) 114 : "r" (p), "r" (v) 115 : "cc", "memory"); 116 #endif 117 } 118 119 static __inline uint32_t 120 atomic_readandclear_32(volatile uint32_t *addr) 121 { 122 uint32_t result,temp; 123 124 #ifdef __GNUC__ 125 __asm __volatile ( 126 "\tsync\n" /* drain writes */ 127 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 128 "li %1, 0\n\t" /* load new value */ 129 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 130 "bne- 1b\n\t" /* spin if failed */ 131 : "=&r"(result), "=&r"(temp), "+m" (*addr) 132 : "r" (addr) 133 : "cc", "memory"); 134 #endif 135 136 return (result); 137 } 138 139 #if 0 140 141 /* 142 * So far I haven't found a way to implement atomic 64-bit ops on the 143 * 32-bit PowerPC without involving major headaches. If anyone has 144 * any ideas, please let me know. =) 145 * - benno@FreeBSD.org 146 */ 147 148 static __inline void 149 atomic_set_64(volatile u_int64_t *p, u_int64_t v) 150 { 151 u_int64_t temp; 152 153 __asm __volatile ( 154 : "=&r" (temp), "=r" (*p) 155 : "r" (*p), "r" (v) 156 : "memory"); 157 } 158 159 static __inline void 160 atomic_clear_64(volatile u_int64_t *p, u_int64_t v) 161 { 162 u_int64_t temp; 163 164 __asm __volatile ( 165 : "=&r" (temp), "=r" (*p) 166 : "r" (*p), "r" (v) 167 : "memory"); 168 } 169 170 static __inline void 171 atomic_add_64(volatile u_int64_t *p, u_int64_t v) 172 { 173 u_int64_t temp; 174 175 __asm __volatile ( 176 : "=&r" (temp), "=r" (*p) 177 : "r" (*p), "r" (v) 178 : "memory"); 179 } 180 181 static __inline void 182 atomic_subtract_64(volatile u_int64_t *p, u_int64_t v) 183 { 184 u_int64_t temp; 185 186 __asm __volatile ( 187 : "=&r" (temp), "=r" (*p) 188 : "r" (*p), "r" (v) 189 : "memory"); 190 } 191 192 static __inline u_int64_t 193 atomic_readandclear_64(volatile u_int64_t *addr) 194 { 195 u_int64_t result,temp; 196 197 __asm __volatile ( 198 : "=&r"(result), "=&r"(temp), "=r" (*addr) 199 : "r"(*addr) 200 : "memory"); 201 202 return result; 203 } 204 205 #endif /* 0 */ 206 207 #define atomic_set_char atomic_set_8 208 #define atomic_clear_char atomic_clear_8 209 #define atomic_add_char atomic_add_8 210 #define atomic_subtract_char atomic_subtract_8 211 212 #define atomic_set_short atomic_set_16 213 #define atomic_clear_short atomic_clear_16 214 #define atomic_add_short atomic_add_16 215 #define atomic_subtract_short atomic_subtract_16 216 217 #define atomic_set_int atomic_set_32 218 #define atomic_clear_int atomic_clear_32 219 #define atomic_add_int atomic_add_32 220 #define atomic_subtract_int atomic_subtract_32 221 #define atomic_readandclear_int atomic_readandclear_32 222 223 #define atomic_set_long atomic_set_32 224 #define atomic_clear_long atomic_clear_32 225 #define atomic_add_long(p, v) atomic_add_32((uint32_t *)p, (uint32_t)v) 226 #define atomic_subtract_long(p, v) atomic_subtract_32((uint32_t *)p, (uint32_t)v) 227 #define atomic_readandclear_long atomic_readandclear_32 228 229 #if 0 230 231 /* See above. */ 232 233 #define atomic_set_long_long atomic_set_64 234 #define atomic_clear_long_long atomic_clear_64 235 #define atomic_add_long_long atomic_add_64 236 #define atomic_subtract_long_long atomic_subtract_64 237 #define atomic_readandclear_long_long atomic_readandclear_64 238 239 #endif /* 0 */ 240 241 #define ATOMIC_ACQ_REL(NAME, WIDTH, TYPE) \ 242 static __inline void \ 243 atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 244 { \ 245 atomic_##NAME##_##WIDTH(p, v); \ 246 powerpc_mb(); \ 247 } \ 248 \ 249 static __inline void \ 250 atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 251 { \ 252 powerpc_mb(); \ 253 atomic_##NAME##_##WIDTH(p, v); \ 254 } \ 255 \ 256 static __inline void \ 257 atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 258 { \ 259 atomic_##NAME##_##WIDTH(p, v); \ 260 powerpc_mb(); \ 261 } \ 262 \ 263 static __inline void \ 264 atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 265 { \ 266 powerpc_mb(); \ 267 atomic_##NAME##_##WIDTH(p, v); \ 268 } 269 270 ATOMIC_ACQ_REL(set, 8, char) 271 ATOMIC_ACQ_REL(clear, 8, char) 272 ATOMIC_ACQ_REL(add, 8, char) 273 ATOMIC_ACQ_REL(subtract, 8, char) 274 ATOMIC_ACQ_REL(set, 16, short) 275 ATOMIC_ACQ_REL(clear, 16, short) 276 ATOMIC_ACQ_REL(add, 16, short) 277 ATOMIC_ACQ_REL(subtract, 16, short) 278 ATOMIC_ACQ_REL(set, 32, int) 279 ATOMIC_ACQ_REL(clear, 32, int) 280 ATOMIC_ACQ_REL(add, 32, int) 281 ATOMIC_ACQ_REL(subtract, 32, int) 282 283 #define atomic_set_acq_long atomic_set_acq_32 284 #define atomic_set_rel_long atomic_set_rel_32 285 #define atomic_clear_acq_long atomic_clear_acq_32 286 #define atomic_clear_rel_long atomic_clear_rel_32 287 #define atomic_add_acq_long atomic_add_acq_32 288 #define atomic_add_rel_long atomic_add_rel_32 289 #define atomic_subtract_acq_long atomic_subtract_acq_32 290 #define atomic_subtract_rel_long atomic_subtract_rel_32 291 292 #undef ATOMIC_ACQ_REL 293 294 /* 295 * We assume that a = b will do atomic loads and stores. 296 */ 297 #define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ 298 static __inline u_##TYPE \ 299 atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ 300 { \ 301 u_##TYPE v; \ 302 \ 303 v = *p; \ 304 powerpc_mb(); \ 305 return (v); \ 306 } \ 307 \ 308 static __inline void \ 309 atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ 310 { \ 311 powerpc_mb(); \ 312 *p = v; \ 313 } \ 314 \ 315 static __inline u_##TYPE \ 316 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 317 { \ 318 u_##TYPE v; \ 319 \ 320 v = *p; \ 321 powerpc_mb(); \ 322 return (v); \ 323 } \ 324 \ 325 static __inline void \ 326 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 327 { \ 328 powerpc_mb(); \ 329 *p = v; \ 330 } 331 332 ATOMIC_STORE_LOAD(char, 8) 333 ATOMIC_STORE_LOAD(short, 16) 334 ATOMIC_STORE_LOAD(int, 32) 335 336 #define atomic_load_acq_long atomic_load_acq_32 337 #define atomic_store_rel_long atomic_store_rel_32 338 339 #undef ATOMIC_STORE_LOAD 340 341 /* 342 * Atomically compare the value stored at *p with cmpval and if the 343 * two values are equal, update the value of *p with newval. Returns 344 * zero if the compare failed, nonzero otherwise. 345 */ 346 static __inline uint32_t 347 atomic_cmpset_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) 348 { 349 uint32_t ret; 350 351 #ifdef __GNUC__ 352 __asm __volatile ( 353 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 354 "cmplw %3, %0\n\t" /* compare */ 355 "bne 2f\n\t" /* exit if not equal */ 356 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 357 "bne- 1b\n\t" /* spin if failed */ 358 "li %0, 1\n\t" /* success - retval = 1 */ 359 "b 3f\n\t" /* we've succeeded */ 360 "2:\n\t" 361 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 362 "li %0, 0\n\t" /* failure - retval = 0 */ 363 "3:\n\t" 364 : "=&r" (ret), "+m" (*p) 365 : "r" (p), "r" (cmpval), "r" (newval) 366 : "cc", "memory"); 367 #endif 368 369 return (ret); 370 } 371 372 #if 0 373 374 /* 375 * Atomically compare the value stored at *p with cmpval and if the 376 * two values are equal, update the value of *p with newval. Returns 377 * zero if the compare failed, nonzero otherwise. 378 */ 379 static __inline u_int64_t 380 atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 381 { 382 u_int64_t ret; 383 384 __asm __volatile ( 385 : "=&r" (ret), "=r" (*p) 386 : "r" (cmpval), "r" (newval), "r" (*p) 387 : "memory"); 388 389 return ret; 390 } 391 392 #endif /* 0 */ 393 394 #define atomic_cmpset_int atomic_cmpset_32 395 #define atomic_cmpset_long atomic_cmpset_32 396 397 #if 0 398 #define atomic_cmpset_long_long atomic_cmpset_64 399 #endif /* 0 */ 400 401 static __inline int 402 atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) 403 { 404 405 return (atomic_cmpset_32((volatile uint32_t *)dst, (uint32_t)exp, 406 (uint32_t)src)); 407 } 408 409 static __inline uint32_t 410 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 411 { 412 int retval; 413 414 retval = atomic_cmpset_32(p, cmpval, newval); 415 powerpc_mb(); 416 return (retval); 417 } 418 419 static __inline uint32_t 420 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 421 { 422 powerpc_mb(); 423 return (atomic_cmpset_32(p, cmpval, newval)); 424 } 425 426 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 427 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 428 #define atomic_cmpset_acq_long atomic_cmpset_acq_32 429 #define atomic_cmpset_rel_long atomic_cmpset_rel_32 430 431 static __inline int 432 atomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src) 433 { 434 435 return (atomic_cmpset_acq_32((volatile uint32_t *)dst, 436 (uint32_t)exp, (uint32_t)src)); 437 } 438 439 static __inline int 440 atomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src) 441 { 442 443 return (atomic_cmpset_rel_32((volatile uint32_t *)dst, 444 (uint32_t)exp, (uint32_t)src)); 445 } 446 447 static __inline void * 448 atomic_load_acq_ptr(volatile void *p) 449 { 450 451 return (void *)atomic_load_acq_32((volatile uint32_t *)p); 452 } 453 454 static __inline void 455 atomic_store_rel_ptr(volatile void *p, void *v) 456 { 457 458 atomic_store_rel_32((volatile uint32_t *)p, (uint32_t)v); 459 } 460 461 #define ATOMIC_PTR(NAME) \ 462 static __inline void \ 463 atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ 464 { \ 465 atomic_##NAME##_32((volatile uint32_t *)p, v); \ 466 } \ 467 \ 468 static __inline void \ 469 atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ 470 { \ 471 atomic_##NAME##_acq_32((volatile uint32_t *)p, v); \ 472 } \ 473 \ 474 static __inline void \ 475 atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ 476 { \ 477 atomic_##NAME##_rel_32((volatile uint32_t *)p, v); \ 478 } 479 480 ATOMIC_PTR(set) 481 ATOMIC_PTR(clear) 482 ATOMIC_PTR(add) 483 ATOMIC_PTR(subtract) 484 485 #undef ATOMIC_PTR 486 #endif /* ! _MACHINE_ATOMIC_H_ */ 487