1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_ATOMIC_H 28 #define _SYS_ATOMIC_H 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #include <sys/types.h> 33 #include <sys/inttypes.h> 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) 40 #include <asm/atomic.h> 41 #endif 42 43 #if defined(_KERNEL) || defined(__STDC__) 44 /* 45 * Increment target. 46 */ 47 extern void atomic_inc_8(volatile uint8_t *); 48 extern void atomic_inc_uchar(volatile uchar_t *); 49 extern void atomic_inc_16(volatile uint16_t *); 50 extern void atomic_inc_ushort(volatile ushort_t *); 51 extern void atomic_inc_32(volatile uint32_t *); 52 extern void atomic_inc_uint(volatile uint_t *); 53 extern void atomic_inc_ulong(volatile ulong_t *); 54 #if defined(_KERNEL) || defined(_INT64_TYPE) 55 extern void atomic_inc_64(volatile uint64_t *); 56 #endif 57 58 /* 59 * Decrement target 60 */ 61 extern void atomic_dec_8(volatile uint8_t *); 62 extern void atomic_dec_uchar(volatile uchar_t *); 63 extern void atomic_dec_16(volatile uint16_t *); 64 extern void atomic_dec_ushort(volatile ushort_t *); 65 extern void atomic_dec_32(volatile uint32_t *); 66 extern void atomic_dec_uint(volatile uint_t *); 67 extern void atomic_dec_ulong(volatile ulong_t *); 68 #if defined(_KERNEL) || defined(_INT64_TYPE) 69 extern void atomic_dec_64(volatile uint64_t *); 70 #endif 71 72 /* 73 * Add delta to target 74 */ 75 extern void atomic_add_8(volatile uint8_t *, int8_t); 76 extern void atomic_add_char(volatile uchar_t *, signed char); 77 extern void atomic_add_16(volatile uint16_t *, int16_t); 78 extern void atomic_add_short(volatile ushort_t *, short); 79 extern void atomic_add_32(volatile uint32_t *, int32_t); 80 extern void atomic_add_int(volatile uint_t *, int); 81 extern void atomic_add_ptr(volatile void *, ssize_t); 82 extern void atomic_add_long(volatile ulong_t *, long); 83 #if defined(_KERNEL) || defined(_INT64_TYPE) 84 extern void atomic_add_64(volatile uint64_t *, int64_t); 85 #endif 86 87 /* 88 * logical OR bits with target 89 */ 90 extern void atomic_or_8(volatile uint8_t *, uint8_t); 91 extern void atomic_or_uchar(volatile uchar_t *, uchar_t); 92 extern void atomic_or_16(volatile uint16_t *, uint16_t); 93 extern void atomic_or_ushort(volatile ushort_t *, ushort_t); 94 extern void atomic_or_32(volatile uint32_t *, uint32_t); 95 extern void atomic_or_uint(volatile uint_t *, uint_t); 96 extern void atomic_or_ulong(volatile ulong_t *, ulong_t); 97 #if defined(_KERNEL) || defined(_INT64_TYPE) 98 extern void atomic_or_64(volatile uint64_t *, uint64_t); 99 #endif 100 101 /* 102 * logical AND bits with target 103 */ 104 extern void atomic_and_8(volatile uint8_t *, uint8_t); 105 extern void atomic_and_uchar(volatile uchar_t *, uchar_t); 106 extern void atomic_and_16(volatile uint16_t *, uint16_t); 107 extern void atomic_and_ushort(volatile ushort_t *, ushort_t); 108 extern void atomic_and_32(volatile uint32_t *, uint32_t); 109 extern void atomic_and_uint(volatile uint_t *, uint_t); 110 extern void atomic_and_ulong(volatile ulong_t *, ulong_t); 111 #if defined(_KERNEL) || defined(_INT64_TYPE) 112 extern void atomic_and_64(volatile uint64_t *, uint64_t); 113 #endif 114 115 /* 116 * As above, but return the new value. Note that these _nv() variants are 117 * substantially more expensive on some platforms than the no-return-value 118 * versions above, so don't use them unless you really need to know the 119 * new value *atomically* (e.g. when decrementing a reference count and 120 * checking whether it went to zero). 121 */ 122 123 /* 124 * Increment target and return new value. 125 */ 126 extern uint8_t atomic_inc_8_nv(volatile uint8_t *); 127 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *); 128 extern uint16_t atomic_inc_16_nv(volatile uint16_t *); 129 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *); 130 extern uint32_t atomic_inc_32_nv(volatile uint32_t *); 131 extern uint_t atomic_inc_uint_nv(volatile uint_t *); 132 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *); 133 #if defined(_KERNEL) || defined(_INT64_TYPE) 134 extern uint64_t atomic_inc_64_nv(volatile uint64_t *); 135 #endif 136 137 /* 138 * Decrement target and return new value. 139 */ 140 extern uint8_t atomic_dec_8_nv(volatile uint8_t *); 141 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *); 142 extern uint16_t atomic_dec_16_nv(volatile uint16_t *); 143 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *); 144 extern uint32_t atomic_dec_32_nv(volatile uint32_t *); 145 extern uint_t atomic_dec_uint_nv(volatile uint_t *); 146 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *); 147 #if defined(_KERNEL) || defined(_INT64_TYPE) 148 extern uint64_t atomic_dec_64_nv(volatile uint64_t *); 149 #endif 150 151 /* 152 * Add delta to target 153 */ 154 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t); 155 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char); 156 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t); 157 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short); 158 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t); 159 extern uint_t atomic_add_int_nv(volatile uint_t *, int); 160 extern void *atomic_add_ptr_nv(volatile void *, ssize_t); 161 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long); 162 #if defined(_KERNEL) || defined(_INT64_TYPE) 163 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t); 164 #endif 165 166 /* 167 * logical OR bits with target and return new value. 168 */ 169 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t); 170 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t); 171 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t); 172 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t); 173 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t); 174 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t); 175 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t); 176 #if defined(_KERNEL) || defined(_INT64_TYPE) 177 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t); 178 #endif 179 180 /* 181 * logical AND bits with target and return new value. 182 */ 183 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t); 184 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t); 185 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t); 186 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t); 187 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t); 188 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t); 189 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t); 190 #if defined(_KERNEL) || defined(_INT64_TYPE) 191 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t); 192 #endif 193 194 /* 195 * If *arg1 == arg2, set *arg1 = arg3; return old value 196 */ 197 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t); 198 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t); 199 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t); 200 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t); 201 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t); 202 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t); 203 extern void *atomic_cas_ptr(volatile void *, void *, void *); 204 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t); 205 #if defined(_KERNEL) || defined(_INT64_TYPE) 206 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t); 207 #endif 208 209 /* 210 * Swap target and return old value 211 */ 212 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t); 213 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t); 214 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t); 215 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t); 216 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t); 217 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t); 218 extern void *atomic_swap_ptr(volatile void *, void *); 219 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t); 220 #if defined(_KERNEL) || defined(_INT64_TYPE) 221 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t); 222 #endif 223 224 /* 225 * Perform an exclusive atomic bit set/clear on a target. 226 * Returns 0 if bit was sucessfully set/cleared, or -1 227 * if the bit was already set/cleared. 228 */ 229 extern int atomic_set_long_excl(volatile ulong_t *, uint_t); 230 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t); 231 232 /* 233 * Generic memory barrier used during lock entry, placed after the 234 * memory operation that acquires the lock to guarantee that the lock 235 * protects its data. No stores from after the memory barrier will 236 * reach visibility, and no loads from after the barrier will be 237 * resolved, before the lock acquisition reaches global visibility. 238 */ 239 extern void membar_enter(void); 240 241 /* 242 * Generic memory barrier used during lock exit, placed before the 243 * memory operation that releases the lock to guarantee that the lock 244 * protects its data. All loads and stores issued before the barrier 245 * will be resolved before the subsequent lock update reaches visibility. 246 */ 247 extern void membar_exit(void); 248 249 /* 250 * Arrange that all stores issued before this point in the code reach 251 * global visibility before any stores that follow; useful in producer 252 * modules that update a data item, then set a flag that it is available. 253 * The memory barrier guarantees that the available flag is not visible 254 * earlier than the updated data, i.e. it imposes store ordering. 255 */ 256 extern void membar_producer(void); 257 258 /* 259 * Arrange that all loads issued before this point in the code are 260 * completed before any subsequent loads; useful in consumer modules 261 * that check to see if data is available and read the data. 262 * The memory barrier guarantees that the data is not sampled until 263 * after the available flag has been seen, i.e. it imposes load ordering. 264 */ 265 extern void membar_consumer(void); 266 #endif 267 268 #if !defined(_KERNEL) && !defined(__STDC__) 269 extern void atomic_inc_8(); 270 extern void atomic_inc_uchar(); 271 extern void atomic_inc_16(); 272 extern void atomic_inc_ushort(); 273 extern void atomic_inc_32(); 274 extern void atomic_inc_uint(); 275 extern void atomic_inc_ulong(); 276 #if defined(_INT64_TYPE) 277 extern void atomic_inc_64(); 278 #endif /* defined(_INT64_TYPE) */ 279 extern void atomic_dec_8(); 280 extern void atomic_dec_uchar(); 281 extern void atomic_dec_16(); 282 extern void atomic_dec_ushort(); 283 extern void atomic_dec_32(); 284 extern void atomic_dec_uint(); 285 extern void atomic_dec_ulong(); 286 #if defined(_INT64_TYPE) 287 extern void atomic_dec_64(); 288 #endif /* defined(_INT64_TYPE) */ 289 extern void atomic_add_8(); 290 extern void atomic_add_char(); 291 extern void atomic_add_16(); 292 extern void atomic_add_short(); 293 extern void atomic_add_32(); 294 extern void atomic_add_int(); 295 extern void atomic_add_ptr(); 296 extern void atomic_add_long(); 297 #if defined(_INT64_TYPE) 298 extern void atomic_add_64(); 299 #endif /* defined(_INT64_TYPE) */ 300 extern void atomic_or_8(); 301 extern void atomic_or_uchar(); 302 extern void atomic_or_16(); 303 extern void atomic_or_ushort(); 304 extern void atomic_or_32(); 305 extern void atomic_or_uint(); 306 extern void atomic_or_ulong(); 307 #if defined(_INT64_TYPE) 308 extern void atomic_or_64(); 309 #endif /* defined(_INT64_TYPE) */ 310 extern void atomic_and_8(); 311 extern void atomic_and_uchar(); 312 extern void atomic_and_16(); 313 extern void atomic_and_ushort(); 314 extern void atomic_and_32(); 315 extern void atomic_and_uint(); 316 extern void atomic_and_ulong(); 317 #if defined(_INT64_TYPE) 318 extern void atomic_and_64(); 319 #endif /* defined(_INT64_TYPE) */ 320 extern uint8_t atomic_inc_8_nv(); 321 extern uchar_t atomic_inc_uchar_nv(); 322 extern uint16_t atomic_inc_16_nv(); 323 extern ushort_t atomic_inc_ushort_nv(); 324 extern uint32_t atomic_inc_32_nv(); 325 extern uint_t atomic_inc_uint_nv(); 326 extern ulong_t atomic_inc_ulong_nv(); 327 #if defined(_INT64_TYPE) 328 extern uint64_t atomic_inc_64_nv(); 329 #endif /* defined(_INT64_TYPE) */ 330 extern uint8_t atomic_dec_8_nv(); 331 extern uchar_t atomic_dec_uchar_nv(); 332 extern uint16_t atomic_dec_16_nv(); 333 extern ushort_t atomic_dec_ushort_nv(); 334 extern uint32_t atomic_dec_32_nv(); 335 extern uint_t atomic_dec_uint_nv(); 336 extern ulong_t atomic_dec_ulong_nv(); 337 #if defined(_INT64_TYPE) 338 extern uint64_t atomic_dec_64_nv(); 339 #endif /* defined(_INT64_TYPE) */ 340 extern uint8_t atomic_add_8_nv(); 341 extern uchar_t atomic_add_char_nv(); 342 extern uint16_t atomic_add_16_nv(); 343 extern ushort_t atomic_add_short_nv(); 344 extern uint32_t atomic_add_32_nv(); 345 extern uint_t atomic_add_int_nv(); 346 extern void *atomic_add_ptr_nv(); 347 extern ulong_t atomic_add_long_nv(); 348 #if defined(_INT64_TYPE) 349 extern uint64_t atomic_add_64_nv(); 350 #endif /* defined(_INT64_TYPE) */ 351 extern uint8_t atomic_or_8_nv(); 352 extern uchar_t atomic_or_uchar_nv(); 353 extern uint16_t atomic_or_16_nv(); 354 extern ushort_t atomic_or_ushort_nv(); 355 extern uint32_t atomic_or_32_nv(); 356 extern uint_t atomic_or_uint_nv(); 357 extern ulong_t atomic_or_ulong_nv(); 358 #if defined(_INT64_TYPE) 359 extern uint64_t atomic_or_64_nv(); 360 #endif /* defined(_INT64_TYPE) */ 361 extern uint8_t atomic_and_8_nv(); 362 extern uchar_t atomic_and_uchar_nv(); 363 extern uint16_t atomic_and_16_nv(); 364 extern ushort_t atomic_and_ushort_nv(); 365 extern uint32_t atomic_and_32_nv(); 366 extern uint_t atomic_and_uint_nv(); 367 extern ulong_t atomic_and_ulong_nv(); 368 #if defined(_INT64_TYPE) 369 extern uint64_t atomic_and_64_nv(); 370 #endif /* defined(_INT64_TYPE) */ 371 extern uint8_t atomic_cas_8(); 372 extern uchar_t atomic_cas_uchar(); 373 extern uint16_t atomic_cas_16(); 374 extern ushort_t atomic_cas_ushort(); 375 extern uint32_t atomic_cas_32(); 376 extern uint_t atomic_cas_uint(); 377 extern void *atomic_cas_ptr(); 378 extern ulong_t atomic_cas_ulong(); 379 #if defined(_INT64_TYPE) 380 extern uint64_t atomic_cas_64(); 381 #endif /* defined(_INT64_TYPE) */ 382 extern uint8_t atomic_swap_8(); 383 extern uchar_t atomic_swap_uchar(); 384 extern uint16_t atomic_swap_16(); 385 extern ushort_t atomic_swap_ushort(); 386 extern uint32_t atomic_swap_32(); 387 extern uint_t atomic_swap_uint(); 388 extern void *atomic_swap_ptr(); 389 extern ulong_t atomic_swap_ulong(); 390 #if defined(_INT64_TYPE) 391 extern uint64_t atomic_swap_64(); 392 #endif /* defined(_INT64_TYPE) */ 393 394 395 extern int atomic_set_long_excl(); 396 extern int atomic_clear_long_excl(); 397 398 extern void membar_enter(); 399 extern void membar_exit(); 400 extern void membar_producer(); 401 extern void membar_consumer(); 402 403 #endif 404 405 #if defined(_KERNEL) 406 407 #if defined(_LP64) || defined(_ILP32) 408 #define atomic_add_ip atomic_add_long 409 #define atomic_add_ip_nv atomic_add_long_nv 410 #define casip atomic_cas_ulong 411 #endif 412 413 #if defined(__sparc) 414 extern uint8_t ldstub(uint8_t *); 415 #endif 416 417 /* 418 * Legacy kernel interfaces; they will go away (eventually). 419 */ 420 extern uint8_t cas8(uint8_t *, uint8_t, uint8_t); 421 extern uint32_t cas32(uint32_t *, uint32_t, uint32_t); 422 extern uint64_t cas64(uint64_t *, uint64_t, uint64_t); 423 extern ulong_t caslong(ulong_t *, ulong_t, ulong_t); 424 extern void *casptr(void *, void *, void *); 425 extern void atomic_and_long(ulong_t *, ulong_t); 426 extern void atomic_or_long(ulong_t *, ulong_t); 427 #if defined(__sparc) 428 extern uint32_t swapl(uint32_t *, uint32_t); 429 #endif 430 431 #endif /* _KERNEL */ 432 433 #ifdef __cplusplus 434 } 435 #endif 436 437 #endif /* _SYS_ATOMIC_H */ 438