1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 26 * Copyright 2015 Josef 'Jeff' Sipek <jeffpc@josefsipek.net> 27 */ 28 29 #ifndef _ASM_ATOMIC_H 30 #define _ASM_ATOMIC_H 31 32 #include <sys/ccompile.h> 33 #include <sys/types.h> 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #if !defined(__lint) && defined(__GNUC__) 40 41 /* BEGIN CSTYLED */ 42 /* 43 * This file contains a number of static inline functions implementing 44 * various atomic variable functions. Note that these are *not* all of the 45 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All 46 * possible atomic_* functions are implemented in usr/src/common/atomic in 47 * pure assembly. In the absence of an identically named function in this 48 * header file, any use of the function will result in the compiler emitting 49 * a function call as usual. On the other hand, if an identically named 50 * function exists in this header as a static inline, the compiler will 51 * inline its contents and the linker never sees the symbol reference. We 52 * use this to avoid implementing some of the more complex and less used 53 * functions and instead falling back to function calls. Note that in some 54 * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64 55 * but not i386. 56 */ 57 58 /* 59 * Instruction suffixes for various operand sizes (assuming AMD64) 60 */ 61 #define SUF_8 "b" 62 #define SUF_16 "w" 63 #define SUF_32 "l" 64 #define SUF_64 "q" 65 66 #if defined(__amd64) 67 #define SUF_LONG SUF_64 68 #define SUF_PTR SUF_64 69 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__) 70 #elif defined(__i386) 71 #define SUF_LONG SUF_32 72 #define SUF_PTR SUF_32 73 #define __ATOMIC_OP64(...) 74 #else 75 #error "port me" 76 #endif 77 78 #if defined(__amd64) || defined(__i386) 79 80 #define __ATOMIC_OPXX(fxn, type, op) \ 81 extern __GNU_INLINE void \ 82 fxn(volatile type *target) \ 83 { \ 84 __asm__ __volatile__( \ 85 "lock; " op " %0" \ 86 : "+m" (*target) \ 87 : /* no inputs */ \ 88 : "cc"); \ 89 } 90 91 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8) 92 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16) 93 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32) 94 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64) 95 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8) 96 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16) 97 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32) 98 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG) 99 100 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8) 101 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16) 102 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32) 103 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64) 104 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8) 105 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16) 106 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32) 107 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG) 108 109 #undef __ATOMIC_OPXX 110 111 #define __ATOMIC_OPXX(fxn, type1, type2, op, reg) \ 112 extern __GNU_INLINE void \ 113 fxn(volatile type1 *target, type2 delta) \ 114 { \ 115 __asm__ __volatile__( \ 116 "lock; " op " %1,%0" \ 117 : "+m" (*target) \ 118 : "i" reg (delta) \ 119 : "cc"); \ 120 } 121 122 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8, "q") 123 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16, "r") 124 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32, "r") 125 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64, "r") 126 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8, "q") 127 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16, "r") 128 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32, "r") 129 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG, "r") 130 131 /* 132 * We don't use the above macro here because atomic_add_ptr has an 133 * inconsistent type. The first argument should really be a 'volatile void 134 * **'. 135 */ 136 extern __GNU_INLINE void 137 atomic_add_ptr(volatile void *target, ssize_t delta) 138 { 139 volatile void **tmp = (volatile void **)target; 140 141 __asm__ __volatile__( 142 "lock; add" SUF_PTR " %1,%0" 143 : "+m" (*tmp) 144 : "ir" (delta) 145 : "cc"); 146 } 147 148 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8, "q") 149 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16, "r") 150 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32, "r") 151 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64, "r") 152 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8, "q") 153 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16, "r") 154 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32, "r") 155 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG, "r") 156 157 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8, "q") 158 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16, "r") 159 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32, "r") 160 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64, "r") 161 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8, "q") 162 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16, "r") 163 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32, "r") 164 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG, "r") 165 166 #undef __ATOMIC_OPXX 167 168 #define __ATOMIC_OPXX(fxn, type, op, reg) \ 169 extern __GNU_INLINE type \ 170 fxn(volatile type *target, type cmp, type new) \ 171 { \ 172 type ret; \ 173 __asm__ __volatile__( \ 174 "lock; " op " %2,%0" \ 175 : "+m" (*target), "=a" (ret) \ 176 : reg (new), "1" (cmp) \ 177 : "cc"); \ 178 return (ret); \ 179 } 180 181 __ATOMIC_OPXX(atomic_cas_8, uint8_t, "cmpxchg" SUF_8, "q") 182 __ATOMIC_OPXX(atomic_cas_16, uint16_t, "cmpxchg" SUF_16, "r") 183 __ATOMIC_OPXX(atomic_cas_32, uint32_t, "cmpxchg" SUF_32, "r") 184 __ATOMIC_OP64(atomic_cas_64, uint64_t, "cmpxchg" SUF_64, "r") 185 __ATOMIC_OPXX(atomic_cas_uchar, uchar_t, "cmpxchg" SUF_8, "q") 186 __ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16, "r") 187 __ATOMIC_OPXX(atomic_cas_uint, uint_t, "cmpxchg" SUF_32, "r") 188 __ATOMIC_OPXX(atomic_cas_ulong, ulong_t, "cmpxchg" SUF_LONG, "r") 189 190 #undef __ATOMIC_OPXX 191 192 /* 193 * We don't use the above macro here because atomic_cas_ptr has an 194 * inconsistent type. The first argument should really be a 'volatile void 195 * **'. 196 */ 197 extern __GNU_INLINE void * 198 atomic_cas_ptr(volatile void *target, void *cmp, void *new) 199 { 200 volatile void **tmp = (volatile void **)target; 201 void *ret; 202 203 __asm__ __volatile__( 204 "lock; cmpxchg" SUF_PTR " %2,%0" 205 : "+m" (*tmp), "=a" (ret) 206 : "r" (new), "1" (cmp) 207 : "cc"); 208 209 return (ret); 210 } 211 212 #define __ATOMIC_OPXX(fxn, type, op, reg) \ 213 extern __GNU_INLINE type \ 214 fxn(volatile type *target, type val) \ 215 { \ 216 __asm__ __volatile__( \ 217 op " %1,%0" \ 218 : "+m" (*target), "+" reg (val)); \ 219 return (val); \ 220 } 221 222 __ATOMIC_OPXX(atomic_swap_8, uint8_t, "xchg" SUF_8, "q") 223 __ATOMIC_OPXX(atomic_swap_16, uint16_t, "xchg" SUF_16, "r") 224 __ATOMIC_OPXX(atomic_swap_32, uint32_t, "xchg" SUF_32, "r") 225 __ATOMIC_OP64(atomic_swap_64, uint64_t, "xchg" SUF_64, "r") 226 __ATOMIC_OPXX(atomic_swap_uchar, uchar_t, "xchg" SUF_8, "q") 227 __ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16, "r") 228 __ATOMIC_OPXX(atomic_swap_uint, uint_t, "xchg" SUF_32, "r") 229 __ATOMIC_OPXX(atomic_swap_ulong, ulong_t, "xchg" SUF_LONG, "r") 230 231 #undef __ATOMIC_OPXX 232 233 /* 234 * We don't use the above macro here because atomic_swap_ptr has an 235 * inconsistent type. The first argument should really be a 'volatile void 236 * **'. 237 */ 238 extern __GNU_INLINE void * 239 atomic_swap_ptr(volatile void *target, void *val) 240 { 241 volatile void **tmp = (volatile void **)target; 242 243 __asm__ __volatile__( 244 "xchg" SUF_PTR " %1,%0" 245 : "+m" (*tmp), "+r" (val)); 246 247 return (val); 248 } 249 250 #else 251 #error "port me" 252 #endif 253 254 #undef SUF_8 255 #undef SUF_16 256 #undef SUF_32 257 #undef SUF_64 258 #undef SUF_LONG 259 #undef SUF_PTR 260 261 #undef __ATOMIC_OP64 262 263 /* END CSTYLED */ 264 265 #endif /* !__lint && __GNUC__ */ 266 267 #ifdef __cplusplus 268 } 269 #endif 270 271 #endif /* _ASM_ATOMIC_H */ 272