1 /*- 2 * Copyright (c) 2001 Benno Rice 3 * Copyright (c) 2001 David E. O'Brien 4 * Copyright (c) 1998 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #ifndef _MACHINE_ATOMIC_H_ 32 #define _MACHINE_ATOMIC_H_ 33 34 #include <machine/cpufunc.h> 35 36 /* 37 * Various simple arithmetic on memory which is atomic in the presence 38 * of interrupts and SMP safe. 39 */ 40 41 void atomic_set_8(volatile u_int8_t *, u_int8_t); 42 void atomic_clear_8(volatile u_int8_t *, u_int8_t); 43 void atomic_add_8(volatile u_int8_t *, u_int8_t); 44 void atomic_subtract_8(volatile u_int8_t *, u_int8_t); 45 46 void atomic_set_16(volatile u_int16_t *, u_int16_t); 47 void atomic_clear_16(volatile u_int16_t *, u_int16_t); 48 void atomic_add_16(volatile u_int16_t *, u_int16_t); 49 void atomic_subtract_16(volatile u_int16_t *, u_int16_t); 50 51 static __inline void 52 atomic_set_32(volatile u_int32_t *p, u_int32_t v) 53 { 54 u_int32_t temp; 55 56 __asm __volatile ( 57 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 58 "or %0, %0, %3\n\t" /* calculate new value */ 59 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 60 "bne- 1\n\t" /* spin if failed */ 61 "eieio\n" /* drain to memory */ 62 : "=&r" (temp), "=r" (*p) 63 : "r" (*p), "r" (v) 64 : "memory"); 65 } 66 67 static __inline void 68 atomic_clear_32(volatile u_int32_t *p, u_int32_t v) 69 { 70 u_int32_t temp; 71 72 __asm __volatile ( 73 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 74 "andc %0, %0, %3\n\t" /* calculate new value */ 75 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 76 "bne- 1\n\t" /* spin if failed */ 77 "eieio\n" /* drain to memory */ 78 : "=&r" (temp), "=r" (*p) 79 : "r" (*p), "r" (v) 80 : "memory"); 81 } 82 83 static __inline void 84 atomic_add_32(volatile u_int32_t *p, u_int32_t v) 85 { 86 u_int32_t temp; 87 88 __asm __volatile ( 89 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 90 "add %0, %0, %3\n\t" /* calculate new value */ 91 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 92 "bne- 1\n\t" /* spin if failed */ 93 "eieio\n" /* Old McDonald had a farm */ 94 : "=&r" (temp), "=r" (*p) 95 : "r" (*p), "r" (v) 96 : "memory"); 97 } 98 99 static __inline void 100 atomic_subtract_32(volatile u_int32_t *p, u_int32_t v) 101 { 102 u_int32_t temp; 103 104 __asm __volatile ( 105 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 106 "sub %0, %3, %0\n\t" /* calculate new value */ 107 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 108 "bne- 1\n\t" /* spin if failed */ 109 "eieio\n" /* drain to memory */ 110 : "=&r" (temp), "=r" (*p) 111 : "r" (*p), "r" (v) 112 : "memory"); 113 } 114 115 static __inline u_int32_t 116 atomic_readandclear_32(volatile u_int32_t *addr) 117 { 118 u_int32_t result,temp; 119 120 __asm __volatile ( 121 "\teieio\n" /* memory barrier */ 122 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 123 "li %1, 0\n\t" /* load new value */ 124 "stwcx. %1, 0, %2\n\t" /* attempt to store */ 125 "bne- 1\n\t" /* spin if failed */ 126 "eieio\n" /* drain to memory */ 127 : "=&r"(result), "=&r"(temp), "=r" (*addr) 128 : "r"(*addr) 129 : "memory"); 130 131 return result; 132 } 133 134 #if 0 135 136 /* 137 * So far I haven't found a way to implement atomic 64-bit ops on the 138 * 32-bit PowerPC without involving major headaches. If anyone has 139 * any ideas, please let me know. =) 140 * - benno@FreeBSD.org 141 */ 142 143 static __inline void 144 atomic_set_64(volatile u_int64_t *p, u_int64_t v) 145 { 146 u_int64_t temp; 147 148 __asm __volatile ( 149 : "=&r" (temp), "=r" (*p) 150 : "r" (*p), "r" (v) 151 : "memory"); 152 } 153 154 static __inline void 155 atomic_clear_64(volatile u_int64_t *p, u_int64_t v) 156 { 157 u_int64_t temp; 158 159 __asm __volatile ( 160 : "=&r" (temp), "=r" (*p) 161 : "r" (*p), "r" (v) 162 : "memory"); 163 } 164 165 static __inline void 166 atomic_add_64(volatile u_int64_t *p, u_int64_t v) 167 { 168 u_int64_t temp; 169 170 __asm __volatile ( 171 : "=&r" (temp), "=r" (*p) 172 : "r" (*p), "r" (v) 173 : "memory"); 174 } 175 176 static __inline void 177 atomic_subtract_64(volatile u_int64_t *p, u_int64_t v) 178 { 179 u_int64_t temp; 180 181 __asm __volatile ( 182 : "=&r" (temp), "=r" (*p) 183 : "r" (*p), "r" (v) 184 : "memory"); 185 } 186 187 static __inline u_int64_t 188 atomic_readandclear_64(volatile u_int64_t *addr) 189 { 190 u_int64_t result,temp; 191 192 __asm __volatile ( 193 : "=&r"(result), "=&r"(temp), "=r" (*addr) 194 : "r"(*addr) 195 : "memory"); 196 197 return result; 198 } 199 200 #endif /* 0 */ 201 202 #define atomic_set_char atomic_set_8 203 #define atomic_clear_char atomic_clear_8 204 #define atomic_add_char atomic_add_8 205 #define atomic_subtract_char atomic_subtract_8 206 207 #define atomic_set_short atomic_set_16 208 #define atomic_clear_short atomic_clear_16 209 #define atomic_add_short atomic_add_16 210 #define atomic_subtract_short atomic_subtract_16 211 212 #define atomic_set_int atomic_set_32 213 #define atomic_clear_int atomic_clear_32 214 #define atomic_add_int atomic_add_32 215 #define atomic_subtract_int atomic_subtract_32 216 #define atomic_readandclear_int atomic_readandclear_32 217 218 #define atomic_set_long atomic_set_32 219 #define atomic_clear_long atomic_clear_32 220 #define atomic_add_long atomic_add_32 221 #define atomic_subtract_long atomic_subtract_32 222 #define atomic_readandclear_long atomic_readandclear_32 223 224 #if 0 225 226 /* See above. */ 227 228 #define atomic_set_long_long atomic_set_64 229 #define atomic_clear_long_long atomic_clear_64 230 #define atomic_add_long_long atomic_add_64 231 #define atomic_subtract_long_long atomic_subtract_64 232 #define atomic_readandclear_long_long atomic_readandclear_64 233 234 #endif /* 0 */ 235 236 #define ATOMIC_ACQ_REL(NAME, WIDTH, TYPE) \ 237 static __inline void \ 238 atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 239 { \ 240 powerpc_mb(); \ 241 atomic_##NAME##_##WIDTH(p, v); \ 242 } \ 243 \ 244 static __inline void \ 245 atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 246 { \ 247 atomic_##NAME##_##WIDTH(p, v); \ 248 powerpc_mb(); \ 249 } \ 250 \ 251 static __inline void \ 252 atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 253 { \ 254 powerpc_mb(); \ 255 atomic_##NAME##_##WIDTH(p, v); \ 256 } \ 257 \ 258 static __inline void \ 259 atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \ 260 { \ 261 atomic_##NAME##_##WIDTH(p, v); \ 262 powerpc_mb(); \ 263 } 264 265 ATOMIC_ACQ_REL(set, 8, char) 266 ATOMIC_ACQ_REL(clear, 8, char) 267 ATOMIC_ACQ_REL(add, 8, char) 268 ATOMIC_ACQ_REL(subtract, 8, char) 269 ATOMIC_ACQ_REL(set, 16, short) 270 ATOMIC_ACQ_REL(clear, 16, short) 271 ATOMIC_ACQ_REL(add, 16, short) 272 ATOMIC_ACQ_REL(subtract, 16, short) 273 ATOMIC_ACQ_REL(set, 32, int) 274 ATOMIC_ACQ_REL(clear, 32, int) 275 ATOMIC_ACQ_REL(add, 32, int) 276 ATOMIC_ACQ_REL(subtract, 32, int) 277 278 #define atomic_set_acq_long atomic_set_acq_32 279 #define atomic_set_rel_long atomic_set_rel_32 280 #define atomic_clear_acq_long atomic_clear_acq_32 281 #define atomic_clear_rel_long atomic_clear_rel_32 282 #define atomic_add_acq_long atomic_add_acq_32 283 #define atomic_add_rel_long atomic_add_rel_32 284 #define atomic_subtract_acq_long atomic_subtract_acq_32 285 #define atomic_subtract_rel_long atomic_subtract_rel_32 286 287 #undef ATOMIC_ACQ_REL 288 289 /* 290 * We assume that a = b will do atomic loads and stores. 291 */ 292 #define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ 293 static __inline u_##TYPE \ 294 atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ 295 { \ 296 powerpc_mb(); \ 297 return (*p); \ 298 } \ 299 \ 300 static __inline void \ 301 atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ 302 { \ 303 *p = v; \ 304 powerpc_mb(); \ 305 } \ 306 static __inline u_##TYPE \ 307 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 308 { \ 309 powerpc_mb(); \ 310 return (*p); \ 311 } \ 312 \ 313 static __inline void \ 314 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 315 { \ 316 *p = v; \ 317 powerpc_mb(); \ 318 } 319 320 ATOMIC_STORE_LOAD(char, 8) 321 ATOMIC_STORE_LOAD(short, 16) 322 ATOMIC_STORE_LOAD(int, 32) 323 324 #define atomic_load_acq_long atomic_load_acq_32 325 #define atomic_store_rel_long atomic_store_rel_32 326 327 #undef ATOMIC_STORE_LOAD 328 329 /* 330 * Atomically compare the value stored at *p with cmpval and if the 331 * two values are equal, update the value of *p with newval. Returns 332 * zero if the compare failed, nonzero otherwise. 333 */ 334 static __inline u_int32_t 335 atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) 336 { 337 u_int32_t ret; 338 339 __asm __volatile ( 340 "1:\tlwarx %0, 0, %4\n\t" /* load old value */ 341 "cmplw 0, %2, %0\n\t" /* compare */ 342 "bne 2\n\t" /* exit if not equal */ 343 "mr %0, %3\n\t" /* value to store */ 344 "stwcx. %0, 0, %1\n\t" /* attempt to store */ 345 "bne- 1\n\t" /* spin if failed */ 346 "eieio\n" /* memory barrier */ 347 "2:\t\n" 348 : "=&r" (ret), "=r" (*p) 349 : "r" (cmpval), "r" (newval), "r" (*p) 350 : "memory"); 351 352 return ret; 353 } 354 355 #if 0 356 357 /* 358 * Atomically compare the value stored at *p with cmpval and if the 359 * two values are equal, update the value of *p with newval. Returns 360 * zero if the compare failed, nonzero otherwise. 361 */ 362 static __inline u_int64_t 363 atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 364 { 365 u_int64_t ret; 366 367 __asm __volatile ( 368 : "=&r" (ret), "=r" (*p) 369 : "r" (cmpval), "r" (newval), "r" (*p) 370 : "memory"); 371 372 return ret; 373 } 374 375 #endif /* 0 */ 376 377 #define atomic_cmpset_int atomic_cmpset_32 378 #define atomic_cmpset_long atomic_cmpset_32 379 380 #if 0 381 #define atomic_cmpset_long_long atomic_cmpset_64 382 #endif /* 0 */ 383 384 static __inline int 385 atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) 386 { 387 388 return (atomic_cmpset_32((volatile u_int32_t *)dst, (u_int32_t)exp, 389 (u_int32_t)src)); 390 } 391 392 static __inline u_int32_t 393 atomic_cmpset_acq_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) 394 { 395 396 powerpc_mb(); 397 return (atomic_cmpset_32(p, cmpval, newval)); 398 } 399 400 static __inline u_int32_t 401 atomic_cmpset_rel_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) 402 { 403 int retval; 404 405 retval = atomic_cmpset_32(p, cmpval, newval); 406 powerpc_mb(); 407 return (retval); 408 } 409 410 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 411 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 412 #define atomic_cmpset_acq_long atomic_cmpset_acq_32 413 #define atomic_cmpset_rel_long atomic_cmpset_rel_32 414 415 static __inline int 416 atomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src) 417 { 418 419 return (atomic_cmpset_acq_long((volatile u_int32_t *)dst, 420 (u_int32_t)exp, (u_int32_t)src)); 421 } 422 423 static __inline int 424 atomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src) 425 { 426 427 return (atomic_cmpset_rel_long((volatile u_int32_t *)dst, 428 (u_int32_t)exp, (u_int32_t)src)); 429 } 430 431 static __inline void * 432 atomic_load_acq_ptr(volatile void *p) 433 { 434 435 return (void *)atomic_load_acq_long((volatile u_int32_t *)p); 436 } 437 438 static __inline void 439 atomic_store_rel_ptr(volatile void *p, void *v) 440 { 441 442 atomic_store_rel_long((volatile u_int32_t *)p, (u_int32_t)v); 443 } 444 445 #define ATOMIC_PTR(NAME) \ 446 static __inline void \ 447 atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ 448 { \ 449 atomic_##NAME##_long((volatile u_int32_t *)p, v); \ 450 } \ 451 \ 452 static __inline void \ 453 atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ 454 { \ 455 atomic_##NAME##_acq_long((volatile u_int32_t *)p, v); \ 456 } \ 457 \ 458 static __inline void \ 459 atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ 460 { \ 461 atomic_##NAME##_rel_long((volatile u_int32_t *)p, v); \ 462 } 463 464 ATOMIC_PTR(set) 465 ATOMIC_PTR(clear) 466 ATOMIC_PTR(add) 467 ATOMIC_PTR(subtract) 468 469 #undef ATOMIC_PTR 470 #endif /* ! _MACHINE_ATOMIC_H_ */ 471