1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 26 */ 27 28 #ifndef _ASM_ATOMIC_H 29 #define _ASM_ATOMIC_H 30 31 #include <sys/ccompile.h> 32 #include <sys/types.h> 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 #if !defined(__lint) && defined(__GNUC__) 39 40 /* BEGIN CSTYLED */ 41 /* 42 * This file contains a number of static inline functions implementing 43 * various atomic variable functions. Note that these are *not* all of the 44 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All 45 * possible atomic_* functions are implemented in usr/src/common/atomic in 46 * pure assembly. In the absence of an identically named function in this 47 * header file, any use of the function will result in the compiler emitting 48 * a function call as usual. On the other hand, if an identically named 49 * function exists in this header as a static inline, the compiler will 50 * inline its contents and the linker never sees the symbol reference. We 51 * use this to avoid implementing some of the more complex and less used 52 * functions and instead falling back to function calls. Note that in some 53 * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64 54 * but not i386. 55 */ 56 57 /* 58 * Instruction suffixes for various operand sizes (assuming AMD64) 59 */ 60 #define SUF_8 "b" 61 #define SUF_16 "w" 62 #define SUF_32 "l" 63 #define SUF_64 "q" 64 65 #if defined(__amd64) 66 #define SUF_LONG SUF_64 67 #define SUF_PTR SUF_64 68 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__) 69 #elif defined(__i386) 70 #define SUF_LONG SUF_32 71 #define SUF_PTR SUF_32 72 #define __ATOMIC_OP64(...) 73 #else 74 #error "port me" 75 #endif 76 77 #if defined(__amd64) || defined(__i386) 78 79 #define __ATOMIC_OPXX(fxn, type, op) \ 80 extern __GNU_INLINE void \ 81 fxn(volatile type *target) \ 82 { \ 83 __asm__ __volatile__( \ 84 "lock; " op " %0" \ 85 : "+m" (*target)); \ 86 } 87 88 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8) 89 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16) 90 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32) 91 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64) 92 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8) 93 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16) 94 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32) 95 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG) 96 97 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8) 98 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16) 99 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32) 100 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64) 101 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8) 102 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16) 103 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32) 104 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG) 105 106 #undef __ATOMIC_OPXX 107 108 #define __ATOMIC_OPXX(fxn, type1, type2, op, reg) \ 109 extern __GNU_INLINE void \ 110 fxn(volatile type1 *target, type2 delta) \ 111 { \ 112 __asm__ __volatile__( \ 113 "lock; " op " %1,%0" \ 114 : "+m" (*target) \ 115 : "i" reg (delta)); \ 116 } 117 118 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8, "q") 119 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16, "r") 120 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32, "r") 121 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64, "r") 122 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8, "q") 123 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16, "r") 124 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32, "r") 125 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG, "r") 126 127 /* 128 * We don't use the above macro here because atomic_add_ptr has an 129 * inconsistent type. The first argument should really be a 'volatile void 130 * **'. 131 */ 132 extern __GNU_INLINE void 133 atomic_add_ptr(volatile void *target, ssize_t delta) 134 { 135 volatile void **tmp = (volatile void **)target; 136 137 __asm__ __volatile__( 138 "lock; add" SUF_PTR " %1,%0" 139 : "+m" (*tmp) 140 : "ir" (delta)); 141 } 142 143 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8, "q") 144 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16, "r") 145 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32, "r") 146 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64, "r") 147 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8, "q") 148 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16, "r") 149 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32, "r") 150 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG, "r") 151 152 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8, "q") 153 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16, "r") 154 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32, "r") 155 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64, "r") 156 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8, "q") 157 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16, "r") 158 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32, "r") 159 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG, "r") 160 161 #undef __ATOMIC_OPXX 162 163 #define __ATOMIC_OPXX(fxn, type, op, reg) \ 164 extern __GNU_INLINE type \ 165 fxn(volatile type *target, type cmp, type new) \ 166 { \ 167 type ret; \ 168 __asm__ __volatile__( \ 169 "lock; " op " %2,%0" \ 170 : "+m" (*target), "=a" (ret) \ 171 : reg (new), "1" (cmp) \ 172 : "cc"); \ 173 return (ret); \ 174 } 175 176 __ATOMIC_OPXX(atomic_cas_8, uint8_t, "cmpxchg" SUF_8, "q") 177 __ATOMIC_OPXX(atomic_cas_16, uint16_t, "cmpxchg" SUF_16, "r") 178 __ATOMIC_OPXX(atomic_cas_32, uint32_t, "cmpxchg" SUF_32, "r") 179 __ATOMIC_OP64(atomic_cas_64, uint64_t, "cmpxchg" SUF_64, "r") 180 __ATOMIC_OPXX(atomic_cas_uchar, uchar_t, "cmpxchg" SUF_8, "q") 181 __ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16, "r") 182 __ATOMIC_OPXX(atomic_cas_uint, uint_t, "cmpxchg" SUF_32, "r") 183 __ATOMIC_OPXX(atomic_cas_ulong, ulong_t, "cmpxchg" SUF_LONG, "r") 184 185 #undef __ATOMIC_OPXX 186 187 /* 188 * We don't use the above macro here because atomic_cas_ptr has an 189 * inconsistent type. The first argument should really be a 'volatile void 190 * **'. 191 */ 192 extern __GNU_INLINE void * 193 atomic_cas_ptr(volatile void *target, void *cmp, void *new) 194 { 195 volatile void **tmp = (volatile void **)target; 196 void *ret; 197 198 __asm__ __volatile__( 199 "lock; cmpxchg" SUF_PTR " %2,%0" 200 : "+m" (*tmp), "=a" (ret) 201 : "r" (new), "1" (cmp) 202 : "cc"); 203 204 return (ret); 205 } 206 207 #define __ATOMIC_OPXX(fxn, type, op, reg) \ 208 extern __GNU_INLINE type \ 209 fxn(volatile type *target, type val) \ 210 { \ 211 __asm__ __volatile__( \ 212 op " %1,%0" \ 213 : "+m" (*target), "+" reg (val)); \ 214 return (val); \ 215 } 216 217 __ATOMIC_OPXX(atomic_swap_8, uint8_t, "xchg" SUF_8, "q") 218 __ATOMIC_OPXX(atomic_swap_16, uint16_t, "xchg" SUF_16, "r") 219 __ATOMIC_OPXX(atomic_swap_32, uint32_t, "xchg" SUF_32, "r") 220 __ATOMIC_OP64(atomic_swap_64, uint64_t, "xchg" SUF_64, "r") 221 __ATOMIC_OPXX(atomic_swap_uchar, uchar_t, "xchg" SUF_8, "q") 222 __ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16, "r") 223 __ATOMIC_OPXX(atomic_swap_uint, uint_t, "xchg" SUF_32, "r") 224 __ATOMIC_OPXX(atomic_swap_ulong, ulong_t, "xchg" SUF_LONG, "r") 225 226 #undef __ATOMIC_OPXX 227 228 /* 229 * We don't use the above macro here because atomic_swap_ptr has an 230 * inconsistent type. The first argument should really be a 'volatile void 231 * **'. 232 */ 233 extern __GNU_INLINE void * 234 atomic_swap_ptr(volatile void *target, void *val) 235 { 236 volatile void **tmp = (volatile void **)target; 237 238 __asm__ __volatile__( 239 "xchg" SUF_PTR " %1,%0" 240 : "+m" (*tmp), "+r" (val)); 241 242 return (val); 243 } 244 245 #else 246 #error "port me" 247 #endif 248 249 #undef SUF_8 250 #undef SUF_16 251 #undef SUF_32 252 #undef SUF_64 253 #undef SUF_LONG 254 #undef SUF_PTR 255 256 #undef __ATOMIC_OP64 257 258 /* END CSTYLED */ 259 260 #endif /* !__lint && __GNUC__ */ 261 262 #ifdef __cplusplus 263 } 264 #endif 265 266 #endif /* _ASM_ATOMIC_H */ 267