1 /*- 2 * Copyright (c) 2001 Benno Rice 3 * Copyright (c) 2001 David E. O'Brien 4 * Copyright (c) 1998 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #ifndef _MACHINE_ATOMIC_H_ 32 #define _MACHINE_ATOMIC_H_ 33 34 #include <machine/cpufunc.h> 35 36 /* 37 * Various simple arithmetic on memory which is atomic in the presence 38 * of interrupts and SMP safe. 39 */ 40 41 void atomic_set_8(volatile u_int8_t *, u_int8_t); 42 void atomic_clear_8(volatile u_int8_t *, u_int8_t); 43 void atomic_add_8(volatile u_int8_t *, u_int8_t); 44 void atomic_subtract_8(volatile u_int8_t *, u_int8_t); 45 46 void atomic_set_16(volatile u_int16_t *, u_int16_t); 47 void atomic_clear_16(volatile u_int16_t *, u_int16_t); 48 void atomic_add_16(volatile u_int16_t *, u_int16_t); 49 void atomic_subtract_16(volatile u_int16_t *, u_int16_t); 50 51 static __inline void 52 atomic_set_32(volatile u_int32_t *p, u_int32_t v) 53 { 54 u_int32_t temp; 55 56 __asm __volatile ( 57 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 58 "or %0, %0, %2\n\t" /* calculate new value */ 59 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 60 "bne- 1\n\t" /* spin if failed */ 61 "eieio\n" /* drain to memory */ 62 : "=&r" (temp) 63 : "r" (p), "r" (v) 64 : "memory"); 65 } 66 67 static __inline void 68 atomic_clear_32(volatile u_int32_t *p, u_int32_t v) 69 { 70 u_int32_t temp; 71 72 __asm __volatile ( 73 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 74 "andc %0, %0, %2\n\t" /* calculate new value */ 75 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 76 "bne- 1\n\t" /* spin if failed */ 77 "eieio\n" /* drain to memory */ 78 : "=&r" (temp) 79 : "r" (p), "r" (v) 80 : "memory"); 81 } 82 83 static __inline void 84 atomic_add_32(volatile u_int32_t *p, u_int32_t v) 85 { 86 u_int32_t temp; 87 88 __asm __volatile ( 89 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 90 "add %0, %0, %2\n\t" /* calculate new value */ 91 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 92 "bne- 1\n\t" /* spin if failed */ 93 "eieio\n" /* Old McDonald had a farm */ 94 : "=&r" (temp) 95 : "r" (p), "r" (v) 96 : "memory"); 97 } 98 99 static __inline void 100 atomic_subtract_32(volatile u_int32_t *p, u_int32_t v) 101 { 102 u_int32_t temp; 103 104 __asm __volatile ( 105 "1:\tlwarx %0, 0, %1\n\t" /* load old value */ 106 "sub %0, %2, %0\n\t" /* calculate new value */ 107 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 108 "bne- 1\n\t" /* spin if failed */ 109 "eieio\n" /* drain to memory */ 110 : "=&r" (temp) 111 : "r" (p), "r" (v) 112 : "memory"); 113 } 114 115 static __inline u_int32_t 116 atomic_readandclear_32(volatile u_int32_t *addr) 117 { 118 u_int32_t result,temp; 119 120 __asm __volatile ( 121 "\teieio\n" /* memory barrier */ 122 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 123 "li %1, 0\n\t" /* load new value */ 124 "stwcx. %1, 0, %2\n\t" /* attempt to store */ 125 "bne- 1\n\t" /* spin if failed */ 126 "eieio\n" /* drain to memory */ 127 : "=&r"(result), "=&r"(temp) 128 : "r"(addr) 129 : "memory"); 130 131 return result; 132 } 133 134 #if 0 135 136 /* 137 * So far I haven't found a way to implement atomic 64-bit ops on the 138 * 32-bit PowerPC without involving major headaches. If anyone has 139 * any ideas, please let me know. =) 140 * - benno@FreeBSD.org 141 */ 142 143 static __inline void 144 atomic_set_64(volatile u_int64_t *p, u_int64_t v) 145 { 146 u_int64_t temp; 147 148 __asm __volatile ( 149 : "=&r" (temp), "=r" (*p) 150 : "r" (*p), "r" (v) 151 : "memory"); 152 } 153 154 static __inline void 155 atomic_clear_64(volatile u_int64_t *p, u_int64_t v) 156 { 157 u_int64_t temp; 158 159 __asm __volatile ( 160 : "=&r" (temp), "=r" (*p) 161 : "r" (*p), "r" (v) 162 : "memory"); 163 } 164 165 static __inline void 166 atomic_add_64(volatile u_int64_t *p, u_int64_t v) 167 { 168 u_int64_t temp; 169 170 __asm __volatile ( 171 : "=&r" (temp), "=r" (*p) 172 : "r" (*p), "r" (v) 173 : "memory"); 174 } 175 176 static __inline void 177 atomic_subtract_64(volatile u_int64_t *p, u_int64_t v) 178 { 179 u_int64_t temp; 180 181 __asm __volatile ( 182 : "=&r" (temp), "=r" (*p) 183 : "r" (*p), "r" (v) 184 : "memory"); 185 } 186 187 static __inline u_int64_t 188 atomic_readandclear_64(volatile u_int64_t *addr) 189 { 190 u_int64_t result,temp; 191 192 __asm __volatile ( 193 : "=&r"(result), "=&r"(temp), "=r" (*addr) 194 : "r"(*addr) 195 : "memory"); 196 197 return result; 198 } 199 200 #endif /* 0 */ 201 202 #define atomic_set_char atomic_set_8 203 #define atomic_clear_char atomic_clear_8 204 #define atomic_add_char atomic_add_8 205 #define atomic_subtract_char atomic_subtract_8 206 207 #define atomic_set_short atomic_set_16 208 #define atomic_clear_short atomic_clear_16 209 #define atomic_add_short atomic_add_16 210 #define atomic_subtract_short atomic_subtract_16 211 212 #define atomic_set_int atomic_set_32 213 #define atomic_clear_int atomic_clear_32 214 #define atomic_add_int atomic_add_32 215 #define atomic_subtract_int atomic_subtract_32 216 #define atomic_readandclear_int atomic_readandclear_32 217 218 #define atomic_set_long atomic_set_32 219 #define atomic_clear_long atomic_clear_32 220 #define atomic_add_long(p, v) atomic_add_32((u_int32_t *)p, (u_int32_t)v) 221 #define atomic_subtract_long atomic_subtract_32 222 #define atomic_readandclear_long atomic_readandclear_32 223 224 #if 0 225 226 /* See above. */ 227 228 #define atomic_set_long_long atomic_set_64 229 #define atomic_clear_long_long atomic_clear_64 230 #define atomic_add_long_long atomic_add_64 231 #define atomic_subtract_long_long atomic_subtract_64 232 #define atomic_readandclear_long_long atomic_readandclear_64 233 234 #endif /* 0 */ 235 236 #define ATOMIC_ACQ_REL(NAME, WIDTH, TYPE) \ 237 static __inline void \ 238 atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 239 { \ 240 powerpc_mb(); \ 241 atomic_##NAME##_##WIDTH(p, v); \ 242 } \ 243 \ 244 static __inline void \ 245 atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 246 { \ 247 atomic_##NAME##_##WIDTH(p, v); \ 248 powerpc_mb(); \ 249 } \ 250 \ 251 static __inline void \ 252 atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 253 { \ 254 powerpc_mb(); \ 255 atomic_##NAME##_##WIDTH(p, v); \ 256 } \ 257 \ 258 static __inline void \ 259 atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 260 { \ 261 atomic_##NAME##_##WIDTH(p, v); \ 262 powerpc_mb(); \ 263 } 264 265 ATOMIC_ACQ_REL(set, 8, char) 266 ATOMIC_ACQ_REL(clear, 8, char) 267 ATOMIC_ACQ_REL(add, 8, char) 268 ATOMIC_ACQ_REL(subtract, 8, char) 269 ATOMIC_ACQ_REL(set, 16, short) 270 ATOMIC_ACQ_REL(clear, 16, short) 271 ATOMIC_ACQ_REL(add, 16, short) 272 ATOMIC_ACQ_REL(subtract, 16, short) 273 ATOMIC_ACQ_REL(set, 32, int) 274 ATOMIC_ACQ_REL(clear, 32, int) 275 ATOMIC_ACQ_REL(add, 32, int) 276 ATOMIC_ACQ_REL(subtract, 32, int) 277 278 #define atomic_set_acq_long atomic_set_acq_32 279 #define atomic_set_rel_long atomic_set_rel_32 280 #define atomic_clear_acq_long atomic_clear_acq_32 281 #define atomic_clear_rel_long atomic_clear_rel_32 282 #define atomic_add_acq_long atomic_add_acq_32 283 #define atomic_add_rel_long atomic_add_rel_32 284 #define atomic_subtract_acq_long atomic_subtract_acq_32 285 #define atomic_subtract_rel_long atomic_subtract_rel_32 286 287 #undef ATOMIC_ACQ_REL 288 289 /* 290 * We assume that a = b will do atomic loads and stores. 291 */ 292 #define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ 293 static __inline u_##TYPE \ 294 atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ 295 { \ 296 powerpc_mb(); \ 297 return (*p); \ 298 } \ 299 \ 300 static __inline void \ 301 atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ 302 { \ 303 *p = v; \ 304 powerpc_mb(); \ 305 } \ 306 static __inline u_##TYPE \ 307 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 308 { \ 309 powerpc_mb(); \ 310 return (*p); \ 311 } \ 312 \ 313 static __inline void \ 314 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 315 { \ 316 *p = v; \ 317 powerpc_mb(); \ 318 } 319 320 ATOMIC_STORE_LOAD(char, 8) 321 ATOMIC_STORE_LOAD(short, 16) 322 ATOMIC_STORE_LOAD(int, 32) 323 324 #define atomic_load_acq_long atomic_load_acq_32 325 #define atomic_store_rel_long atomic_store_rel_32 326 327 #undef ATOMIC_STORE_LOAD 328 329 /* 330 * Atomically compare the value stored at *p with cmpval and if the 331 * two values are equal, update the value of *p with newval. Returns 332 * zero if the compare failed, nonzero otherwise. 333 */ 334 static __inline u_int32_t 335 atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) 336 { 337 u_int32_t ret; 338 339 __asm __volatile ( 340 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 341 "cmplw 0, %1, %0\n\t" /* compare */ 342 "bne 2f\n\t" /* exit if not equal */ 343 "mr %0, %2\n\t" /* value to store */ 344 "stwcx. %0, 0, %3\n\t" /* attempt to store */ 345 "bne- 1b\n\t" /* spin if failed */ 346 "eieio\n" /* memory barrier */ 347 "sync\n" 348 "2:\t\n" 349 : "=&r" (ret) 350 : "r" (cmpval), "r" (newval), "r" (p) 351 : "memory"); 352 353 return ret; 354 } 355 356 #if 0 357 358 /* 359 * Atomically compare the value stored at *p with cmpval and if the 360 * two values are equal, update the value of *p with newval. Returns 361 * zero if the compare failed, nonzero otherwise. 362 */ 363 static __inline u_int64_t 364 atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 365 { 366 u_int64_t ret; 367 368 __asm __volatile ( 369 : "=&r" (ret), "=r" (*p) 370 : "r" (cmpval), "r" (newval), "r" (*p) 371 : "memory"); 372 373 return ret; 374 } 375 376 #endif /* 0 */ 377 378 #define atomic_cmpset_int atomic_cmpset_32 379 #define atomic_cmpset_long atomic_cmpset_32 380 381 #if 0 382 #define atomic_cmpset_long_long atomic_cmpset_64 383 #endif /* 0 */ 384 385 static __inline int 386 atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) 387 { 388 389 return (atomic_cmpset_32((volatile u_int32_t *)dst, (u_int32_t)exp, 390 (u_int32_t)src)); 391 } 392 393 static __inline u_int32_t 394 atomic_cmpset_acq_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) 395 { 396 397 powerpc_mb(); 398 return (atomic_cmpset_32(p, cmpval, newval)); 399 } 400 401 static __inline u_int32_t 402 atomic_cmpset_rel_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) 403 { 404 int retval; 405 406 retval = atomic_cmpset_32(p, cmpval, newval); 407 powerpc_mb(); 408 return (retval); 409 } 410 411 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 412 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 413 #define atomic_cmpset_acq_long atomic_cmpset_acq_32 414 #define atomic_cmpset_rel_long atomic_cmpset_rel_32 415 416 static __inline int 417 atomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src) 418 { 419 420 return (atomic_cmpset_acq_32((volatile u_int32_t *)dst, 421 (u_int32_t)exp, (u_int32_t)src)); 422 } 423 424 static __inline int 425 atomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src) 426 { 427 428 return (atomic_cmpset_rel_32((volatile u_int32_t *)dst, 429 (u_int32_t)exp, (u_int32_t)src)); 430 } 431 432 static __inline void * 433 atomic_load_acq_ptr(volatile void *p) 434 { 435 436 return (void *)atomic_load_acq_32((volatile u_int32_t *)p); 437 } 438 439 static __inline void 440 atomic_store_rel_ptr(volatile void *p, void *v) 441 { 442 443 atomic_store_rel_32((volatile u_int32_t *)p, (u_int32_t)v); 444 } 445 446 #define ATOMIC_PTR(NAME) \ 447 static __inline void \ 448 atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ 449 { \ 450 atomic_##NAME##_32((volatile u_int32_t *)p, v); \ 451 } \ 452 \ 453 static __inline void \ 454 atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ 455 { \ 456 atomic_##NAME##_acq_32((volatile u_int32_t *)p, v); \ 457 } \ 458 \ 459 static __inline void \ 460 atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ 461 { \ 462 atomic_##NAME##_rel_32((volatile u_int32_t *)p, v); \ 463 } 464 465 ATOMIC_PTR(set) 466 ATOMIC_PTR(clear) 467 ATOMIC_PTR(add) 468 ATOMIC_PTR(subtract) 469 470 #undef ATOMIC_PTR 471 #endif /* ! _MACHINE_ATOMIC_H_ */ 472