1 /*- 2 * Copyright (c) 2001 Benno Rice 3 * Copyright (c) 2001 David E. O'Brien 4 * Copyright (c) 1998 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #ifndef _MACHINE_ATOMIC_H_ 32 #define _MACHINE_ATOMIC_H_ 33 34 #include <machine/cpufunc.h> 35 36 /* 37 * Various simple arithmetic on memory which is atomic in the presence 38 * of interrupts and SMP safe. 39 */ 40 41 void atomic_set_8(volatile u_int8_t *, u_int8_t); 42 void atomic_clear_8(volatile u_int8_t *, u_int8_t); 43 void atomic_add_8(volatile u_int8_t *, u_int8_t); 44 void atomic_subtract_8(volatile u_int8_t *, u_int8_t); 45 46 void atomic_set_16(volatile u_int16_t *, u_int16_t); 47 void atomic_clear_16(volatile u_int16_t *, u_int16_t); 48 void atomic_add_16(volatile u_int16_t *, u_int16_t); 49 void atomic_subtract_16(volatile u_int16_t *, u_int16_t); 50 51 static __inline void 52 atomic_set_32(volatile u_int32_t *p, u_int32_t v) 53 { 54 u_int32_t temp; 55 56 __asm __volatile ( 57 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 58 "or %0, %0, %2\n\t" /* calculate new value */ 59 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 60 "bne- 1\n\t" /* spin if failed */ 61 "eieio\n" /* drain to memory */ 62 : "=&r" (temp) 63 : "r" (p), "r" (v) 64 : "memory"); 65 } 66 67 static __inline void 68 atomic_clear_32(volatile u_int32_t *p, u_int32_t v) 69 { 70 u_int32_t temp; 71 72 __asm __volatile ( 73 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 74 "andc %0, %0, %2\n\t" /* calculate new value */ 75 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 76 "bne- 1\n\t" /* spin if failed */ 77 "eieio\n" /* drain to memory */ 78 : "=&r" (temp) 79 : "r" (p), "r" (v) 80 : "memory"); 81 } 82 83 static __inline void 84 atomic_add_32(volatile u_int32_t *p, u_int32_t v) 85 { 86 u_int32_t temp; 87 88 __asm __volatile ( 89 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 90 "add %0, %0, %2\n\t" /* calculate new value */ 91 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 92 "bne- 1\n\t" /* spin if failed */ 93 "eieio\n" /* Old McDonald had a farm */ 94 : "=&r" (temp) 95 : "r" (p), "r" (v) 96 : "memory"); 97 } 98 99 static __inline void 100 atomic_subtract_32(volatile u_int32_t *p, u_int32_t v) 101 { 102 u_int32_t temp; 103 104 __asm __volatile ( 105 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 106 "sub %0, %2, %0\n\t" /* calculate new value */ 107 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 108 "bne- 1\n\t" /* spin if failed */ 109 "eieio\n" /* drain to memory */ 110 : "=&r" (temp) 111 : "r" (p), "r" (v) 112 : "memory"); 113 } 114 115 static __inline u_int32_t 116 atomic_readandclear_32(volatile u_int32_t *addr) 117 { 118 u_int32_t result,temp; 119 120 __asm __volatile ( 121 "\teieio\n" /* memory barrier */ 122 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 123 "li %1, 0\n\t" /* load new value */ 124 "stwcx. %1, 0, %2\n\t" /* attempt to store */ 125 "bne- 1\n\t" /* spin if failed */ 126 "eieio\n" /* drain to memory */ 127 : "=&r"(result), "=&r"(temp) 128 : "r"(addr) 129 : "memory"); 130 131 return result; 132 } 133 134 #if 0 135 136 /* 137 * So far I haven't found a way to implement atomic 64-bit ops on the 138 * 32-bit PowerPC without involving major headaches. If anyone has 139 * any ideas, please let me know. =) 140 * - benno@FreeBSD.org 141 */ 142 143 static __inline void 144 atomic_set_64(volatile u_int64_t *p, u_int64_t v) 145 { 146 u_int64_t temp; 147 148 __asm __volatile ( 149 : "=&r" (temp), "=r" (*p) 150 : "r" (*p), "r" (v) 151 : "memory"); 152 } 153 154 static __inline void 155 atomic_clear_64(volatile u_int64_t *p, u_int64_t v) 156 { 157 u_int64_t temp; 158 159 __asm __volatile ( 160 : "=&r" (temp), "=r" (*p) 161 : "r" (*p), "r" (v) 162 : "memory"); 163 } 164 165 static __inline void 166 atomic_add_64(volatile u_int64_t *p, u_int64_t v) 167 { 168 u_int64_t temp; 169 170 __asm __volatile ( 171 : "=&r" (temp), "=r" (*p) 172 : "r" (*p), "r" (v) 173 : "memory"); 174 } 175 176 static __inline void 177 atomic_subtract_64(volatile u_int64_t *p, u_int64_t v) 178 { 179 u_int64_t temp; 180 181 __asm __volatile ( 182 : "=&r" (temp), "=r" (*p) 183 : "r" (*p), "r" (v) 184 : "memory"); 185 } 186 187 static __inline u_int64_t 188 atomic_readandclear_64(volatile u_int64_t *addr) 189 { 190 u_int64_t result,temp; 191 192 __asm __volatile ( 193 : "=&r"(result), "=&r"(temp), "=r" (*addr) 194 : "r"(*addr) 195 : "memory"); 196 197 return result; 198 } 199 200 #endif /* 0 */ 201 202 #define atomic_set_char atomic_set_8 203 #define atomic_clear_char atomic_clear_8 204 #define atomic_add_char atomic_add_8 205 #define atomic_subtract_char atomic_subtract_8 206 207 #define atomic_set_short atomic_set_16 208 #define atomic_clear_short atomic_clear_16 209 #define atomic_add_short atomic_add_16 210 #define atomic_subtract_short atomic_subtract_16 211 212 #define atomic_set_int atomic_set_32 213 #define atomic_clear_int atomic_clear_32 214 #define atomic_add_int atomic_add_32 215 #define atomic_subtract_int atomic_subtract_32 216 #define atomic_readandclear_int atomic_readandclear_32 217 218 #define atomic_set_long atomic_set_32 219 #define atomic_clear_long atomic_clear_32 220 #define atomic_add_long(p, v) atomic_add_32((u_int32_t *)p, (u_int32_t)v) 221 #define atomic_subtract_long atomic_subtract_32 222 #define atomic_readandclear_long atomic_readandclear_32 223 224 #if 0 225 226 /* See above. */ 227 228 #define atomic_set_long_long atomic_set_64 229 #define atomic_clear_long_long atomic_clear_64 230 #define atomic_add_long_long atomic_add_64 231 #define atomic_subtract_long_long atomic_subtract_64 232 #define atomic_readandclear_long_long atomic_readandclear_64 233 234 #endif /* 0 */ 235 236 #define ATOMIC_ACQ_REL(NAME, WIDTH, TYPE) \ 237 static __inline void \ 238 atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 239 { \ 240 powerpc_mb(); \ 241 atomic_##NAME##_##WIDTH(p, v); \ 242 } \ 243 \ 244 static __inline void \ 245 atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 246 { \ 247 atomic_##NAME##_##WIDTH(p, v); \ 248 powerpc_mb(); \ 249 } \ 250 \ 251 static __inline void \ 252 atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 253 { \ 254 powerpc_mb(); \ 255 atomic_##NAME##_##WIDTH(p, v); \ 256 } \ 257 \ 258 static __inline void \ 259 atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 260 { \ 261 atomic_##NAME##_##WIDTH(p, v); \ 262 powerpc_mb(); \ 263 } 264 265 ATOMIC_ACQ_REL(set, 8, char) 266 ATOMIC_ACQ_REL(clear, 8, char) 267 ATOMIC_ACQ_REL(add, 8, char) 268 ATOMIC_ACQ_REL(subtract, 8, char) 269 ATOMIC_ACQ_REL(set, 16, short) 270 ATOMIC_ACQ_REL(clear, 16, short) 271 ATOMIC_ACQ_REL(add, 16, short) 272 ATOMIC_ACQ_REL(subtract, 16, short) 273 ATOMIC_ACQ_REL(set, 32, int) 274 ATOMIC_ACQ_REL(clear, 32, int) 275 ATOMIC_ACQ_REL(add, 32, int) 276 ATOMIC_ACQ_REL(subtract, 32, int) 277 278 #define atomic_set_acq_long atomic_set_acq_32 279 #define atomic_set_rel_long atomic_set_rel_32 280 #define atomic_clear_acq_long atomic_clear_acq_32 281 #define atomic_clear_rel_long atomic_clear_rel_32 282 #define atomic_add_acq_long atomic_add_acq_32 283 #define atomic_add_rel_long atomic_add_rel_32 284 #define atomic_subtract_acq_long atomic_subtract_acq_32 285 #define atomic_subtract_rel_long atomic_subtract_rel_32 286 287 #undef ATOMIC_ACQ_REL 288 289 /* 290 * We assume that a = b will do atomic loads and stores. 291 */ 292 #define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ 293 static __inline u_##TYPE \ 294 atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ 295 { \ 296 powerpc_mb(); \ 297 return (*p); \ 298 } \ 299 \ 300 static __inline void \ 301 atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ 302 { \ 303 *p = v; \ 304 powerpc_mb(); \ 305 } \ 306 static __inline u_##TYPE \ 307 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 308 { \ 309 powerpc_mb(); \ 310 return (*p); \ 311 } \ 312 \ 313 static __inline void \ 314 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 315 { \ 316 *p = v; \ 317 powerpc_mb(); \ 318 } 319 320 ATOMIC_STORE_LOAD(char, 8) 321 ATOMIC_STORE_LOAD(short, 16) 322 ATOMIC_STORE_LOAD(int, 32) 323 324 #define atomic_load_acq_long atomic_load_acq_32 325 #define atomic_store_rel_long atomic_store_rel_32 326 327 #undef ATOMIC_STORE_LOAD 328 329 /* 330 * Atomically compare the value stored at *p with cmpval and if the 331 * two values are equal, update the value of *p with newval. Returns 332 * zero if the compare failed, nonzero otherwise. 333 */ 334 static __inline u_int32_t 335 atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) 336 { 337 u_int32_t ret; 338 339 __asm __volatile ( 340 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 341 "cmplw 0, %1, %0\n\t" /* compare */ 342 "bne 2f\n\t" /* exit if not equal */ 343 "mr %0, %2\n\t" /* value to store */ 344 "stwcx. %0, 0, %3\n\t" /* attempt to store */ 345 "bne- 1b\n\t" /* spin if failed */ 346 "eieio\n" /* memory barrier */ 347 "sync\n" 348 "b 3f\n\t" /* we've succeeded */ 349 "2:\t\n" 350 "xor %0,%0,%0\t\n" /* failure, so return 0 */ 351 "3:\t\n" 352 : "=&r" (ret) 353 : "r" (cmpval), "r" (newval), "r" (p) 354 : "memory"); 355 356 return ret; 357 } 358 359 #if 0 360 361 /* 362 * Atomically compare the value stored at *p with cmpval and if the 363 * two values are equal, update the value of *p with newval. Returns 364 * zero if the compare failed, nonzero otherwise. 365 */ 366 static __inline u_int64_t 367 atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 368 { 369 u_int64_t ret; 370 371 __asm __volatile ( 372 : "=&r" (ret), "=r" (*p) 373 : "r" (cmpval), "r" (newval), "r" (*p) 374 : "memory"); 375 376 return ret; 377 } 378 379 #endif /* 0 */ 380 381 #define atomic_cmpset_int atomic_cmpset_32 382 #define atomic_cmpset_long atomic_cmpset_32 383 384 #if 0 385 #define atomic_cmpset_long_long atomic_cmpset_64 386 #endif /* 0 */ 387 388 static __inline int 389 atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) 390 { 391 392 return (atomic_cmpset_32((volatile u_int32_t *)dst, (u_int32_t)exp, 393 (u_int32_t)src)); 394 } 395 396 static __inline u_int32_t 397 atomic_cmpset_acq_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) 398 { 399 400 powerpc_mb(); 401 return (atomic_cmpset_32(p, cmpval, newval)); 402 } 403 404 static __inline u_int32_t 405 atomic_cmpset_rel_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) 406 { 407 int retval; 408 409 retval = atomic_cmpset_32(p, cmpval, newval); 410 powerpc_mb(); 411 return (retval); 412 } 413 414 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 415 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 416 #define atomic_cmpset_acq_long atomic_cmpset_acq_32 417 #define atomic_cmpset_rel_long atomic_cmpset_rel_32 418 419 static __inline int 420 atomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src) 421 { 422 423 return (atomic_cmpset_acq_32((volatile u_int32_t *)dst, 424 (u_int32_t)exp, (u_int32_t)src)); 425 } 426 427 static __inline int 428 atomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src) 429 { 430 431 return (atomic_cmpset_rel_32((volatile u_int32_t *)dst, 432 (u_int32_t)exp, (u_int32_t)src)); 433 } 434 435 static __inline void * 436 atomic_load_acq_ptr(volatile void *p) 437 { 438 439 return (void *)atomic_load_acq_32((volatile u_int32_t *)p); 440 } 441 442 static __inline void 443 atomic_store_rel_ptr(volatile void *p, void *v) 444 { 445 446 atomic_store_rel_32((volatile u_int32_t *)p, (u_int32_t)v); 447 } 448 449 #define ATOMIC_PTR(NAME) \ 450 static __inline void \ 451 atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ 452 { \ 453 atomic_##NAME##_32((volatile u_int32_t *)p, v); \ 454 } \ 455 \ 456 static __inline void \ 457 atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ 458 { \ 459 atomic_##NAME##_acq_32((volatile u_int32_t *)p, v); \ 460 } \ 461 \ 462 static __inline void \ 463 atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ 464 { \ 465 atomic_##NAME##_rel_32((volatile u_int32_t *)p, v); \ 466 } 467 468 ATOMIC_PTR(set) 469 ATOMIC_PTR(clear) 470 ATOMIC_PTR(add) 471 ATOMIC_PTR(subtract) 472 473 #undef ATOMIC_PTR 474 #endif /* ! _MACHINE_ATOMIC_H_ */ 475