1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 25 * Use is subject to license terms. 26 */ 27 28 #ifndef _SYS_ATOMIC_H 29 #define _SYS_ATOMIC_H 30 31 #include <sys/types.h> 32 #include <sys/inttypes.h> 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 #if defined(__STDC__) 39 /* 40 * Increment target. 41 */ 42 extern void atomic_inc_8(volatile uint8_t *); 43 extern void atomic_inc_uchar(volatile uchar_t *); 44 extern void atomic_inc_16(volatile uint16_t *); 45 extern void atomic_inc_ushort(volatile ushort_t *); 46 extern void atomic_inc_32(volatile uint32_t *); 47 extern void atomic_inc_uint(volatile uint_t *); 48 extern void atomic_inc_ulong(volatile ulong_t *); 49 #if defined(_INT64_TYPE) 50 extern void atomic_inc_64(volatile uint64_t *); 51 #endif 52 53 /* 54 * Decrement target 55 */ 56 extern void atomic_dec_8(volatile uint8_t *); 57 extern void atomic_dec_uchar(volatile uchar_t *); 58 extern void atomic_dec_16(volatile uint16_t *); 59 extern void atomic_dec_ushort(volatile ushort_t *); 60 extern void atomic_dec_32(volatile uint32_t *); 61 extern void atomic_dec_uint(volatile uint_t *); 62 extern void atomic_dec_ulong(volatile ulong_t *); 63 #if defined(_INT64_TYPE) 64 extern void atomic_dec_64(volatile uint64_t *); 65 #endif 66 67 /* 68 * Add delta to target 69 */ 70 extern void atomic_add_8(volatile uint8_t *, int8_t); 71 extern void atomic_add_char(volatile uchar_t *, signed char); 72 extern void atomic_add_16(volatile uint16_t *, int16_t); 73 extern void atomic_add_short(volatile ushort_t *, short); 74 extern void atomic_add_32(volatile uint32_t *, int32_t); 75 extern void atomic_add_int(volatile uint_t *, int); 76 extern void atomic_add_ptr(volatile void *, ssize_t); 77 extern void atomic_add_long(volatile ulong_t *, long); 78 #if defined(_INT64_TYPE) 79 extern void atomic_add_64(volatile uint64_t *, int64_t); 80 #endif 81 82 /* 83 * Subtract delta from target 84 */ 85 extern void atomic_sub_8(volatile uint8_t *, int8_t); 86 extern void atomic_sub_char(volatile uchar_t *, signed char); 87 extern void atomic_sub_16(volatile uint16_t *, int16_t); 88 extern void atomic_sub_short(volatile ushort_t *, short); 89 extern void atomic_sub_32(volatile uint32_t *, int32_t); 90 extern void atomic_sub_int(volatile uint_t *, int); 91 extern void atomic_sub_ptr(volatile void *, ssize_t); 92 extern void atomic_sub_long(volatile ulong_t *, long); 93 #if defined(_INT64_TYPE) 94 extern void atomic_sub_64(volatile uint64_t *, int64_t); 95 #endif 96 97 /* 98 * logical OR bits with target 99 */ 100 extern void atomic_or_8(volatile uint8_t *, uint8_t); 101 extern void atomic_or_uchar(volatile uchar_t *, uchar_t); 102 extern void atomic_or_16(volatile uint16_t *, uint16_t); 103 extern void atomic_or_ushort(volatile ushort_t *, ushort_t); 104 extern void atomic_or_32(volatile uint32_t *, uint32_t); 105 extern void atomic_or_uint(volatile uint_t *, uint_t); 106 extern void atomic_or_ulong(volatile ulong_t *, ulong_t); 107 #if defined(_INT64_TYPE) 108 extern void atomic_or_64(volatile uint64_t *, uint64_t); 109 #endif 110 111 /* 112 * logical AND bits with target 113 */ 114 extern void atomic_and_8(volatile uint8_t *, uint8_t); 115 extern void atomic_and_uchar(volatile uchar_t *, uchar_t); 116 extern void atomic_and_16(volatile uint16_t *, uint16_t); 117 extern void atomic_and_ushort(volatile ushort_t *, ushort_t); 118 extern void atomic_and_32(volatile uint32_t *, uint32_t); 119 extern void atomic_and_uint(volatile uint_t *, uint_t); 120 extern void atomic_and_ulong(volatile ulong_t *, ulong_t); 121 #if defined(_INT64_TYPE) 122 extern void atomic_and_64(volatile uint64_t *, uint64_t); 123 #endif 124 125 /* 126 * As above, but return the new value. Note that these _nv() variants are 127 * substantially more expensive on some platforms than the no-return-value 128 * versions above, so don't use them unless you really need to know the 129 * new value *atomically* (e.g. when decrementing a reference count and 130 * checking whether it went to zero). 131 */ 132 133 /* 134 * Increment target and return new value. 135 */ 136 extern uint8_t atomic_inc_8_nv(volatile uint8_t *); 137 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *); 138 extern uint16_t atomic_inc_16_nv(volatile uint16_t *); 139 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *); 140 extern uint32_t atomic_inc_32_nv(volatile uint32_t *); 141 extern uint_t atomic_inc_uint_nv(volatile uint_t *); 142 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *); 143 #if defined(_INT64_TYPE) 144 extern uint64_t atomic_inc_64_nv(volatile uint64_t *); 145 #endif 146 147 /* 148 * Decrement target and return new value. 149 */ 150 extern uint8_t atomic_dec_8_nv(volatile uint8_t *); 151 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *); 152 extern uint16_t atomic_dec_16_nv(volatile uint16_t *); 153 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *); 154 extern uint32_t atomic_dec_32_nv(volatile uint32_t *); 155 extern uint_t atomic_dec_uint_nv(volatile uint_t *); 156 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *); 157 #if defined(_INT64_TYPE) 158 extern uint64_t atomic_dec_64_nv(volatile uint64_t *); 159 #endif 160 161 /* 162 * Add delta to target 163 */ 164 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t); 165 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char); 166 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t); 167 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short); 168 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t); 169 extern uint_t atomic_add_int_nv(volatile uint_t *, int); 170 extern void *atomic_add_ptr_nv(volatile void *, ssize_t); 171 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long); 172 #if defined(_INT64_TYPE) 173 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t); 174 #endif 175 176 /* 177 * Subtract delta from target 178 */ 179 extern uint8_t atomic_sub_8_nv(volatile uint8_t *, int8_t); 180 extern uchar_t atomic_sub_char_nv(volatile uchar_t *, signed char); 181 extern uint16_t atomic_sub_16_nv(volatile uint16_t *, int16_t); 182 extern ushort_t atomic_sub_short_nv(volatile ushort_t *, short); 183 extern uint32_t atomic_sub_32_nv(volatile uint32_t *, int32_t); 184 extern uint_t atomic_sub_int_nv(volatile uint_t *, int); 185 extern void *atomic_sub_ptr_nv(volatile void *, ssize_t); 186 extern ulong_t atomic_sub_long_nv(volatile ulong_t *, long); 187 #if defined(_INT64_TYPE) 188 extern uint64_t atomic_sub_64_nv(volatile uint64_t *, int64_t); 189 #endif 190 191 /* 192 * logical OR bits with target and return new value. 193 */ 194 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t); 195 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t); 196 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t); 197 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t); 198 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t); 199 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t); 200 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t); 201 #if defined(_INT64_TYPE) 202 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t); 203 #endif 204 205 /* 206 * logical AND bits with target and return new value. 207 */ 208 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t); 209 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t); 210 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t); 211 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t); 212 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t); 213 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t); 214 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t); 215 #if defined(_INT64_TYPE) 216 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t); 217 #endif 218 219 /* 220 * If *arg1 == arg2, set *arg1 = arg3; return old value 221 */ 222 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t); 223 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t); 224 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t); 225 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t); 226 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t); 227 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t); 228 extern void *atomic_cas_ptr(volatile void *, void *, void *); 229 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t); 230 #if defined(_INT64_TYPE) 231 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t); 232 #endif 233 234 /* 235 * Swap target and return old value 236 */ 237 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t); 238 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t); 239 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t); 240 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t); 241 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t); 242 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t); 243 extern void *atomic_swap_ptr(volatile void *, void *); 244 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t); 245 #if defined(_INT64_TYPE) 246 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t); 247 #endif 248 249 /* 250 * Atomically read variable. 251 */ 252 #define atomic_load_char(p) (*(volatile uchar_t *)(p)) 253 #define atomic_load_short(p) (*(volatile ushort_t *)(p)) 254 #define atomic_load_int(p) (*(volatile uint_t *)(p)) 255 #define atomic_load_long(p) (*(volatile ulong_t *)(p)) 256 #define atomic_load_ptr(p) (*(volatile __typeof(*p) *)(p)) 257 #define atomic_load_8(p) (*(volatile uint8_t *)(p)) 258 #define atomic_load_16(p) (*(volatile uint16_t *)(p)) 259 #define atomic_load_32(p) (*(volatile uint32_t *)(p)) 260 #ifdef _LP64 261 #define atomic_load_64(p) (*(volatile uint64_t *)(p)) 262 #elif defined(_INT64_TYPE) 263 extern uint64_t atomic_load_64(volatile uint64_t *); 264 #endif 265 266 /* 267 * Atomically write variable. 268 */ 269 #define atomic_store_char(p, v) \ 270 (*(volatile uchar_t *)(p) = (uchar_t)(v)) 271 #define atomic_store_short(p, v) \ 272 (*(volatile ushort_t *)(p) = (ushort_t)(v)) 273 #define atomic_store_int(p, v) \ 274 (*(volatile uint_t *)(p) = (uint_t)(v)) 275 #define atomic_store_long(p, v) \ 276 (*(volatile ulong_t *)(p) = (ulong_t)(v)) 277 #define atomic_store_ptr(p, v) \ 278 (*(volatile __typeof(*p) *)(p) = (v)) 279 #define atomic_store_8(p, v) \ 280 (*(volatile uint8_t *)(p) = (uint8_t)(v)) 281 #define atomic_store_16(p, v) \ 282 (*(volatile uint16_t *)(p) = (uint16_t)(v)) 283 #define atomic_store_32(p, v) \ 284 (*(volatile uint32_t *)(p) = (uint32_t)(v)) 285 #ifdef _LP64 286 #define atomic_store_64(p, v) \ 287 (*(volatile uint64_t *)(p) = (uint64_t)(v)) 288 #elif defined(_INT64_TYPE) 289 extern void atomic_store_64(volatile uint64_t *, uint64_t); 290 #endif 291 292 /* 293 * Perform an exclusive atomic bit set/clear on a target. 294 * Returns 0 if bit was successfully set/cleared, or -1 295 * if the bit was already set/cleared. 296 */ 297 extern int atomic_set_long_excl(volatile ulong_t *, uint_t); 298 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t); 299 300 /* 301 * Generic memory barrier used during lock entry, placed after the 302 * memory operation that acquires the lock to guarantee that the lock 303 * protects its data. No stores from after the memory barrier will 304 * reach visibility, and no loads from after the barrier will be 305 * resolved, before the lock acquisition reaches global visibility. 306 */ 307 extern void membar_enter(void); 308 309 /* 310 * Generic memory barrier used during lock exit, placed before the 311 * memory operation that releases the lock to guarantee that the lock 312 * protects its data. All loads and stores issued before the barrier 313 * will be resolved before the subsequent lock update reaches visibility. 314 */ 315 extern void membar_exit(void); 316 317 /* 318 * Make all stores and loads emitted prior to the the barrier complete before 319 * crossing it, while also making sure stores and loads emitted after the 320 * barrier only start being executed after crossing it. 321 */ 322 extern void membar_sync(void); 323 324 /* 325 * Arrange that all stores issued before this point in the code reach 326 * global visibility before any stores that follow; useful in producer 327 * modules that update a data item, then set a flag that it is available. 328 * The memory barrier guarantees that the available flag is not visible 329 * earlier than the updated data, i.e. it imposes store ordering. 330 */ 331 extern void membar_producer(void); 332 333 /* 334 * Arrange that all loads issued before this point in the code are 335 * completed before any subsequent loads; useful in consumer modules 336 * that check to see if data is available and read the data. 337 * The memory barrier guarantees that the data is not sampled until 338 * after the available flag has been seen, i.e. it imposes load ordering. 339 */ 340 extern void membar_consumer(void); 341 #endif /* __STDC__ */ 342 343 #ifdef __cplusplus 344 } 345 #endif 346 347 #endif /* _SYS_ATOMIC_H */ 348