1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2014 Garrett D'Amore <garrett@damore.org> 24 * 25 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms. 27 */ 28 29 #ifndef _SYS_ATOMIC_H 30 #define _SYS_ATOMIC_H 31 32 #include <sys/types.h> 33 #include <sys/inttypes.h> 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \ 40 (defined(__i386) || defined(__amd64)) 41 #include <asm/atomic.h> 42 #endif 43 44 /* 45 * Increment target. 46 */ 47 extern void atomic_inc_8(volatile uint8_t *); 48 extern void atomic_inc_uchar(volatile uchar_t *); 49 extern void atomic_inc_16(volatile uint16_t *); 50 extern void atomic_inc_ushort(volatile ushort_t *); 51 extern void atomic_inc_32(volatile uint32_t *); 52 extern void atomic_inc_uint(volatile uint_t *); 53 extern void atomic_inc_ulong(volatile ulong_t *); 54 #if defined(_KERNEL) || defined(_INT64_TYPE) 55 extern void atomic_inc_64(volatile uint64_t *); 56 57 /* 58 * Decrement target 59 */ 60 extern void atomic_dec_8(volatile uint8_t *); 61 extern void atomic_dec_uchar(volatile uchar_t *); 62 extern void atomic_dec_16(volatile uint16_t *); 63 extern void atomic_dec_ushort(volatile ushort_t *); 64 extern void atomic_dec_32(volatile uint32_t *); 65 extern void atomic_dec_uint(volatile uint_t *); 66 extern void atomic_dec_ulong(volatile ulong_t *); 67 #if defined(_KERNEL) || defined(_INT64_TYPE) 68 extern void atomic_dec_64(volatile uint64_t *); 69 #endif 70 71 /* 72 * Add delta to target 73 */ 74 extern void atomic_add_8(volatile uint8_t *, int8_t); 75 extern void atomic_add_char(volatile uchar_t *, signed char); 76 extern void atomic_add_16(volatile uint16_t *, int16_t); 77 extern void atomic_add_short(volatile ushort_t *, short); 78 extern void atomic_add_32(volatile uint32_t *, int32_t); 79 extern void atomic_add_int(volatile uint_t *, int); 80 extern void atomic_add_ptr(volatile void *, ssize_t); 81 extern void atomic_add_long(volatile ulong_t *, long); 82 #if defined(_KERNEL) || defined(_INT64_TYPE) 83 extern void atomic_add_64(volatile uint64_t *, int64_t); 84 #endif 85 86 /* 87 * logical OR bits with target 88 */ 89 extern void atomic_or_8(volatile uint8_t *, uint8_t); 90 extern void atomic_or_uchar(volatile uchar_t *, uchar_t); 91 extern void atomic_or_16(volatile uint16_t *, uint16_t); 92 extern void atomic_or_ushort(volatile ushort_t *, ushort_t); 93 extern void atomic_or_32(volatile uint32_t *, uint32_t); 94 extern void atomic_or_uint(volatile uint_t *, uint_t); 95 extern void atomic_or_ulong(volatile ulong_t *, ulong_t); 96 #if defined(_KERNEL) || defined(_INT64_TYPE) 97 extern void atomic_or_64(volatile uint64_t *, uint64_t); 98 #endif 99 100 /* 101 * logical AND bits with target 102 */ 103 extern void atomic_and_8(volatile uint8_t *, uint8_t); 104 extern void atomic_and_uchar(volatile uchar_t *, uchar_t); 105 extern void atomic_and_16(volatile uint16_t *, uint16_t); 106 extern void atomic_and_ushort(volatile ushort_t *, ushort_t); 107 extern void atomic_and_32(volatile uint32_t *, uint32_t); 108 extern void atomic_and_uint(volatile uint_t *, uint_t); 109 extern void atomic_and_ulong(volatile ulong_t *, ulong_t); 110 #if defined(_KERNEL) || defined(_INT64_TYPE) 111 extern void atomic_and_64(volatile uint64_t *, uint64_t); 112 #endif 113 114 /* 115 * As above, but return the new value. Note that these _nv() variants are 116 * substantially more expensive on some platforms than the no-return-value 117 * versions above, so don't use them unless you really need to know the 118 * new value *atomically* (e.g. when decrementing a reference count and 119 * checking whether it went to zero). 120 */ 121 122 /* 123 * Increment target and return new value. 124 */ 125 extern uint8_t atomic_inc_8_nv(volatile uint8_t *); 126 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *); 127 extern uint16_t atomic_inc_16_nv(volatile uint16_t *); 128 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *); 129 extern uint32_t atomic_inc_32_nv(volatile uint32_t *); 130 extern uint_t atomic_inc_uint_nv(volatile uint_t *); 131 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *); 132 #if defined(_KERNEL) || defined(_INT64_TYPE) 133 extern uint64_t atomic_inc_64_nv(volatile uint64_t *); 134 #endif 135 136 /* 137 * Decrement target and return new value. 138 */ 139 extern uint8_t atomic_dec_8_nv(volatile uint8_t *); 140 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *); 141 extern uint16_t atomic_dec_16_nv(volatile uint16_t *); 142 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *); 143 extern uint32_t atomic_dec_32_nv(volatile uint32_t *); 144 extern uint_t atomic_dec_uint_nv(volatile uint_t *); 145 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *); 146 #if defined(_KERNEL) || defined(_INT64_TYPE) 147 extern uint64_t atomic_dec_64_nv(volatile uint64_t *); 148 #endif 149 150 /* 151 * Add delta to target 152 */ 153 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t); 154 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char); 155 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t); 156 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short); 157 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t); 158 extern uint_t atomic_add_int_nv(volatile uint_t *, int); 159 extern void *atomic_add_ptr_nv(volatile void *, ssize_t); 160 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long); 161 #if defined(_KERNEL) || defined(_INT64_TYPE) 162 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t); 163 #endif 164 165 /* 166 * logical OR bits with target and return new value. 167 */ 168 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t); 169 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t); 170 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t); 171 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t); 172 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t); 173 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t); 174 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t); 175 #if defined(_KERNEL) || defined(_INT64_TYPE) 176 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t); 177 #endif 178 179 /* 180 * logical AND bits with target and return new value. 181 */ 182 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t); 183 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t); 184 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t); 185 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t); 186 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t); 187 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t); 188 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t); 189 #if defined(_KERNEL) || defined(_INT64_TYPE) 190 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t); 191 #endif 192 193 /* 194 * If *arg1 == arg2, set *arg1 = arg3; return old value 195 */ 196 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t); 197 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t); 198 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t); 199 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t); 200 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t); 201 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t); 202 extern void *atomic_cas_ptr(volatile void *, void *, void *); 203 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t); 204 #if defined(_KERNEL) || defined(_INT64_TYPE) 205 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t); 206 #endif 207 208 /* 209 * Swap target and return old value 210 */ 211 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t); 212 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t); 213 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t); 214 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t); 215 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t); 216 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t); 217 extern void *atomic_swap_ptr(volatile void *, void *); 218 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t); 219 #if defined(_KERNEL) || defined(_INT64_TYPE) 220 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t); 221 #endif 222 223 /* 224 * Perform an exclusive atomic bit set/clear on a target. 225 * Returns 0 if bit was sucessfully set/cleared, or -1 226 * if the bit was already set/cleared. 227 */ 228 extern int atomic_set_long_excl(volatile ulong_t *, uint_t); 229 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t); 230 231 /* 232 * Generic memory barrier used during lock entry, placed after the 233 * memory operation that acquires the lock to guarantee that the lock 234 * protects its data. No stores from after the memory barrier will 235 * reach visibility, and no loads from after the barrier will be 236 * resolved, before the lock acquisition reaches global visibility. 237 */ 238 extern void membar_enter(void); 239 240 /* 241 * Generic memory barrier used during lock exit, placed before the 242 * memory operation that releases the lock to guarantee that the lock 243 * protects its data. All loads and stores issued before the barrier 244 * will be resolved before the subsequent lock update reaches visibility. 245 */ 246 extern void membar_exit(void); 247 248 /* 249 * Arrange that all stores issued before this point in the code reach 250 * global visibility before any stores that follow; useful in producer 251 * modules that update a data item, then set a flag that it is available. 252 * The memory barrier guarantees that the available flag is not visible 253 * earlier than the updated data, i.e. it imposes store ordering. 254 */ 255 extern void membar_producer(void); 256 257 /* 258 * Arrange that all loads issued before this point in the code are 259 * completed before any subsequent loads; useful in consumer modules 260 * that check to see if data is available and read the data. 261 * The memory barrier guarantees that the data is not sampled until 262 * after the available flag has been seen, i.e. it imposes load ordering. 263 */ 264 extern void membar_consumer(void); 265 #endif 266 267 #if defined(_KERNEL) 268 269 #if defined(_LP64) || defined(_ILP32) 270 #define atomic_add_ip atomic_add_long 271 #define atomic_add_ip_nv atomic_add_long_nv 272 #define casip atomic_cas_ulong 273 #endif 274 275 #if defined(__sparc) 276 extern uint8_t ldstub(uint8_t *); 277 #endif 278 279 #endif /* _KERNEL */ 280 281 #ifdef __cplusplus 282 } 283 #endif 284 285 #endif /* _SYS_ATOMIC_H */ 286