1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_ATOMIC_H 28 #define _SYS_ATOMIC_H 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #include <sys/types.h> 33 #include <sys/inttypes.h> 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \ 40 (defined(__i386) || defined(__amd64)) 41 #include <asm/atomic.h> 42 #endif 43 44 #if defined(_KERNEL) || defined(__STDC__) 45 /* 46 * Increment target. 47 */ 48 extern void atomic_inc_8(volatile uint8_t *); 49 extern void atomic_inc_uchar(volatile uchar_t *); 50 extern void atomic_inc_16(volatile uint16_t *); 51 extern void atomic_inc_ushort(volatile ushort_t *); 52 extern void atomic_inc_32(volatile uint32_t *); 53 extern void atomic_inc_uint(volatile uint_t *); 54 extern void atomic_inc_ulong(volatile ulong_t *); 55 #if defined(_KERNEL) || defined(_INT64_TYPE) 56 extern void atomic_inc_64(volatile uint64_t *); 57 #endif 58 59 /* 60 * Decrement target 61 */ 62 extern void atomic_dec_8(volatile uint8_t *); 63 extern void atomic_dec_uchar(volatile uchar_t *); 64 extern void atomic_dec_16(volatile uint16_t *); 65 extern void atomic_dec_ushort(volatile ushort_t *); 66 extern void atomic_dec_32(volatile uint32_t *); 67 extern void atomic_dec_uint(volatile uint_t *); 68 extern void atomic_dec_ulong(volatile ulong_t *); 69 #if defined(_KERNEL) || defined(_INT64_TYPE) 70 extern void atomic_dec_64(volatile uint64_t *); 71 #endif 72 73 /* 74 * Add delta to target 75 */ 76 extern void atomic_add_8(volatile uint8_t *, int8_t); 77 extern void atomic_add_char(volatile uchar_t *, signed char); 78 extern void atomic_add_16(volatile uint16_t *, int16_t); 79 extern void atomic_add_short(volatile ushort_t *, short); 80 extern void atomic_add_32(volatile uint32_t *, int32_t); 81 extern void atomic_add_int(volatile uint_t *, int); 82 extern void atomic_add_ptr(volatile void *, ssize_t); 83 extern void atomic_add_long(volatile ulong_t *, long); 84 #if defined(_KERNEL) || defined(_INT64_TYPE) 85 extern void atomic_add_64(volatile uint64_t *, int64_t); 86 #endif 87 88 /* 89 * logical OR bits with target 90 */ 91 extern void atomic_or_8(volatile uint8_t *, uint8_t); 92 extern void atomic_or_uchar(volatile uchar_t *, uchar_t); 93 extern void atomic_or_16(volatile uint16_t *, uint16_t); 94 extern void atomic_or_ushort(volatile ushort_t *, ushort_t); 95 extern void atomic_or_32(volatile uint32_t *, uint32_t); 96 extern void atomic_or_uint(volatile uint_t *, uint_t); 97 extern void atomic_or_ulong(volatile ulong_t *, ulong_t); 98 #if defined(_KERNEL) || defined(_INT64_TYPE) 99 extern void atomic_or_64(volatile uint64_t *, uint64_t); 100 #endif 101 102 /* 103 * logical AND bits with target 104 */ 105 extern void atomic_and_8(volatile uint8_t *, uint8_t); 106 extern void atomic_and_uchar(volatile uchar_t *, uchar_t); 107 extern void atomic_and_16(volatile uint16_t *, uint16_t); 108 extern void atomic_and_ushort(volatile ushort_t *, ushort_t); 109 extern void atomic_and_32(volatile uint32_t *, uint32_t); 110 extern void atomic_and_uint(volatile uint_t *, uint_t); 111 extern void atomic_and_ulong(volatile ulong_t *, ulong_t); 112 #if defined(_KERNEL) || defined(_INT64_TYPE) 113 extern void atomic_and_64(volatile uint64_t *, uint64_t); 114 #endif 115 116 /* 117 * As above, but return the new value. Note that these _nv() variants are 118 * substantially more expensive on some platforms than the no-return-value 119 * versions above, so don't use them unless you really need to know the 120 * new value *atomically* (e.g. when decrementing a reference count and 121 * checking whether it went to zero). 122 */ 123 124 /* 125 * Increment target and return new value. 126 */ 127 extern uint8_t atomic_inc_8_nv(volatile uint8_t *); 128 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *); 129 extern uint16_t atomic_inc_16_nv(volatile uint16_t *); 130 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *); 131 extern uint32_t atomic_inc_32_nv(volatile uint32_t *); 132 extern uint_t atomic_inc_uint_nv(volatile uint_t *); 133 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *); 134 #if defined(_KERNEL) || defined(_INT64_TYPE) 135 extern uint64_t atomic_inc_64_nv(volatile uint64_t *); 136 #endif 137 138 /* 139 * Decrement target and return new value. 140 */ 141 extern uint8_t atomic_dec_8_nv(volatile uint8_t *); 142 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *); 143 extern uint16_t atomic_dec_16_nv(volatile uint16_t *); 144 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *); 145 extern uint32_t atomic_dec_32_nv(volatile uint32_t *); 146 extern uint_t atomic_dec_uint_nv(volatile uint_t *); 147 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *); 148 #if defined(_KERNEL) || defined(_INT64_TYPE) 149 extern uint64_t atomic_dec_64_nv(volatile uint64_t *); 150 #endif 151 152 /* 153 * Add delta to target 154 */ 155 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t); 156 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char); 157 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t); 158 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short); 159 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t); 160 extern uint_t atomic_add_int_nv(volatile uint_t *, int); 161 extern void *atomic_add_ptr_nv(volatile void *, ssize_t); 162 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long); 163 #if defined(_KERNEL) || defined(_INT64_TYPE) 164 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t); 165 #endif 166 167 /* 168 * logical OR bits with target and return new value. 169 */ 170 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t); 171 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t); 172 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t); 173 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t); 174 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t); 175 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t); 176 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t); 177 #if defined(_KERNEL) || defined(_INT64_TYPE) 178 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t); 179 #endif 180 181 /* 182 * logical AND bits with target and return new value. 183 */ 184 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t); 185 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t); 186 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t); 187 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t); 188 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t); 189 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t); 190 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t); 191 #if defined(_KERNEL) || defined(_INT64_TYPE) 192 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t); 193 #endif 194 195 /* 196 * If *arg1 == arg2, set *arg1 = arg3; return old value 197 */ 198 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t); 199 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t); 200 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t); 201 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t); 202 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t); 203 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t); 204 extern void *atomic_cas_ptr(volatile void *, void *, void *); 205 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t); 206 #if defined(_KERNEL) || defined(_INT64_TYPE) 207 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t); 208 #endif 209 210 /* 211 * Swap target and return old value 212 */ 213 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t); 214 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t); 215 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t); 216 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t); 217 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t); 218 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t); 219 extern void *atomic_swap_ptr(volatile void *, void *); 220 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t); 221 #if defined(_KERNEL) || defined(_INT64_TYPE) 222 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t); 223 #endif 224 225 /* 226 * Perform an exclusive atomic bit set/clear on a target. 227 * Returns 0 if bit was sucessfully set/cleared, or -1 228 * if the bit was already set/cleared. 229 */ 230 extern int atomic_set_long_excl(volatile ulong_t *, uint_t); 231 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t); 232 233 /* 234 * Generic memory barrier used during lock entry, placed after the 235 * memory operation that acquires the lock to guarantee that the lock 236 * protects its data. No stores from after the memory barrier will 237 * reach visibility, and no loads from after the barrier will be 238 * resolved, before the lock acquisition reaches global visibility. 239 */ 240 extern void membar_enter(void); 241 242 /* 243 * Generic memory barrier used during lock exit, placed before the 244 * memory operation that releases the lock to guarantee that the lock 245 * protects its data. All loads and stores issued before the barrier 246 * will be resolved before the subsequent lock update reaches visibility. 247 */ 248 extern void membar_exit(void); 249 250 /* 251 * Arrange that all stores issued before this point in the code reach 252 * global visibility before any stores that follow; useful in producer 253 * modules that update a data item, then set a flag that it is available. 254 * The memory barrier guarantees that the available flag is not visible 255 * earlier than the updated data, i.e. it imposes store ordering. 256 */ 257 extern void membar_producer(void); 258 259 /* 260 * Arrange that all loads issued before this point in the code are 261 * completed before any subsequent loads; useful in consumer modules 262 * that check to see if data is available and read the data. 263 * The memory barrier guarantees that the data is not sampled until 264 * after the available flag has been seen, i.e. it imposes load ordering. 265 */ 266 extern void membar_consumer(void); 267 #endif 268 269 #if !defined(_KERNEL) && !defined(__STDC__) 270 extern void atomic_inc_8(); 271 extern void atomic_inc_uchar(); 272 extern void atomic_inc_16(); 273 extern void atomic_inc_ushort(); 274 extern void atomic_inc_32(); 275 extern void atomic_inc_uint(); 276 extern void atomic_inc_ulong(); 277 #if defined(_INT64_TYPE) 278 extern void atomic_inc_64(); 279 #endif /* defined(_INT64_TYPE) */ 280 extern void atomic_dec_8(); 281 extern void atomic_dec_uchar(); 282 extern void atomic_dec_16(); 283 extern void atomic_dec_ushort(); 284 extern void atomic_dec_32(); 285 extern void atomic_dec_uint(); 286 extern void atomic_dec_ulong(); 287 #if defined(_INT64_TYPE) 288 extern void atomic_dec_64(); 289 #endif /* defined(_INT64_TYPE) */ 290 extern void atomic_add_8(); 291 extern void atomic_add_char(); 292 extern void atomic_add_16(); 293 extern void atomic_add_short(); 294 extern void atomic_add_32(); 295 extern void atomic_add_int(); 296 extern void atomic_add_ptr(); 297 extern void atomic_add_long(); 298 #if defined(_INT64_TYPE) 299 extern void atomic_add_64(); 300 #endif /* defined(_INT64_TYPE) */ 301 extern void atomic_or_8(); 302 extern void atomic_or_uchar(); 303 extern void atomic_or_16(); 304 extern void atomic_or_ushort(); 305 extern void atomic_or_32(); 306 extern void atomic_or_uint(); 307 extern void atomic_or_ulong(); 308 #if defined(_INT64_TYPE) 309 extern void atomic_or_64(); 310 #endif /* defined(_INT64_TYPE) */ 311 extern void atomic_and_8(); 312 extern void atomic_and_uchar(); 313 extern void atomic_and_16(); 314 extern void atomic_and_ushort(); 315 extern void atomic_and_32(); 316 extern void atomic_and_uint(); 317 extern void atomic_and_ulong(); 318 #if defined(_INT64_TYPE) 319 extern void atomic_and_64(); 320 #endif /* defined(_INT64_TYPE) */ 321 extern uint8_t atomic_inc_8_nv(); 322 extern uchar_t atomic_inc_uchar_nv(); 323 extern uint16_t atomic_inc_16_nv(); 324 extern ushort_t atomic_inc_ushort_nv(); 325 extern uint32_t atomic_inc_32_nv(); 326 extern uint_t atomic_inc_uint_nv(); 327 extern ulong_t atomic_inc_ulong_nv(); 328 #if defined(_INT64_TYPE) 329 extern uint64_t atomic_inc_64_nv(); 330 #endif /* defined(_INT64_TYPE) */ 331 extern uint8_t atomic_dec_8_nv(); 332 extern uchar_t atomic_dec_uchar_nv(); 333 extern uint16_t atomic_dec_16_nv(); 334 extern ushort_t atomic_dec_ushort_nv(); 335 extern uint32_t atomic_dec_32_nv(); 336 extern uint_t atomic_dec_uint_nv(); 337 extern ulong_t atomic_dec_ulong_nv(); 338 #if defined(_INT64_TYPE) 339 extern uint64_t atomic_dec_64_nv(); 340 #endif /* defined(_INT64_TYPE) */ 341 extern uint8_t atomic_add_8_nv(); 342 extern uchar_t atomic_add_char_nv(); 343 extern uint16_t atomic_add_16_nv(); 344 extern ushort_t atomic_add_short_nv(); 345 extern uint32_t atomic_add_32_nv(); 346 extern uint_t atomic_add_int_nv(); 347 extern void *atomic_add_ptr_nv(); 348 extern ulong_t atomic_add_long_nv(); 349 #if defined(_INT64_TYPE) 350 extern uint64_t atomic_add_64_nv(); 351 #endif /* defined(_INT64_TYPE) */ 352 extern uint8_t atomic_or_8_nv(); 353 extern uchar_t atomic_or_uchar_nv(); 354 extern uint16_t atomic_or_16_nv(); 355 extern ushort_t atomic_or_ushort_nv(); 356 extern uint32_t atomic_or_32_nv(); 357 extern uint_t atomic_or_uint_nv(); 358 extern ulong_t atomic_or_ulong_nv(); 359 #if defined(_INT64_TYPE) 360 extern uint64_t atomic_or_64_nv(); 361 #endif /* defined(_INT64_TYPE) */ 362 extern uint8_t atomic_and_8_nv(); 363 extern uchar_t atomic_and_uchar_nv(); 364 extern uint16_t atomic_and_16_nv(); 365 extern ushort_t atomic_and_ushort_nv(); 366 extern uint32_t atomic_and_32_nv(); 367 extern uint_t atomic_and_uint_nv(); 368 extern ulong_t atomic_and_ulong_nv(); 369 #if defined(_INT64_TYPE) 370 extern uint64_t atomic_and_64_nv(); 371 #endif /* defined(_INT64_TYPE) */ 372 extern uint8_t atomic_cas_8(); 373 extern uchar_t atomic_cas_uchar(); 374 extern uint16_t atomic_cas_16(); 375 extern ushort_t atomic_cas_ushort(); 376 extern uint32_t atomic_cas_32(); 377 extern uint_t atomic_cas_uint(); 378 extern void *atomic_cas_ptr(); 379 extern ulong_t atomic_cas_ulong(); 380 #if defined(_INT64_TYPE) 381 extern uint64_t atomic_cas_64(); 382 #endif /* defined(_INT64_TYPE) */ 383 extern uint8_t atomic_swap_8(); 384 extern uchar_t atomic_swap_uchar(); 385 extern uint16_t atomic_swap_16(); 386 extern ushort_t atomic_swap_ushort(); 387 extern uint32_t atomic_swap_32(); 388 extern uint_t atomic_swap_uint(); 389 extern void *atomic_swap_ptr(); 390 extern ulong_t atomic_swap_ulong(); 391 #if defined(_INT64_TYPE) 392 extern uint64_t atomic_swap_64(); 393 #endif /* defined(_INT64_TYPE) */ 394 395 396 extern int atomic_set_long_excl(); 397 extern int atomic_clear_long_excl(); 398 399 extern void membar_enter(); 400 extern void membar_exit(); 401 extern void membar_producer(); 402 extern void membar_consumer(); 403 404 #endif 405 406 #if defined(_KERNEL) 407 408 #if defined(_LP64) || defined(_ILP32) 409 #define atomic_add_ip atomic_add_long 410 #define atomic_add_ip_nv atomic_add_long_nv 411 #define casip atomic_cas_ulong 412 #endif 413 414 #if defined(__sparc) 415 extern uint8_t ldstub(uint8_t *); 416 #endif 417 418 /* 419 * Legacy kernel interfaces; they will go away (eventually). 420 */ 421 extern uint8_t cas8(uint8_t *, uint8_t, uint8_t); 422 extern uint32_t cas32(uint32_t *, uint32_t, uint32_t); 423 extern uint64_t cas64(uint64_t *, uint64_t, uint64_t); 424 extern ulong_t caslong(ulong_t *, ulong_t, ulong_t); 425 extern void *casptr(void *, void *, void *); 426 extern void atomic_and_long(ulong_t *, ulong_t); 427 extern void atomic_or_long(ulong_t *, ulong_t); 428 #if defined(__sparc) 429 extern uint32_t swapl(uint32_t *, uint32_t); 430 #endif 431 432 #endif /* _KERNEL */ 433 434 #ifdef __cplusplus 435 } 436 #endif 437 438 #endif /* _SYS_ATOMIC_H */ 439