1 /* 2 * Copyright 2016-2026 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License 2.0 (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 #ifndef OSSL_INTERNAL_REFCOUNT_H 10 #define OSSL_INTERNAL_REFCOUNT_H 11 #pragma once 12 13 #include <openssl/e_os2.h> 14 #include <openssl/trace.h> 15 #include <openssl/err.h> 16 17 #if defined(OPENSSL_THREADS) && !defined(OPENSSL_DEV_NO_ATOMICS) 18 #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ 19 && !defined(__STDC_NO_ATOMICS__) 20 #include <stdatomic.h> 21 #define HAVE_C11_ATOMICS 22 #endif 23 24 #if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \ 25 && ATOMIC_INT_LOCK_FREE > 0 26 27 #define HAVE_ATOMICS 1 28 29 #if defined(__has_feature) 30 #if __has_feature(thread_sanitizer) 31 #define OSSL_TSAN_BUILD 32 #endif 33 #endif 34 35 typedef struct { 36 _Atomic int val; 37 } CRYPTO_REF_COUNT; 38 39 static inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 40 { 41 *ret = atomic_fetch_add_explicit(&refcnt->val, 1, memory_order_relaxed) + 1; 42 return 1; 43 } 44 45 /* 46 * Changes to shared structure other than reference counter have to be 47 * serialized. And any kind of serialization implies a release fence. This 48 * means that by the time reference counter is decremented all other 49 * changes are visible on all processors. Hence decrement itself can be 50 * relaxed. In case it hits zero, object will be destructed. Since it's 51 * last use of the object, destructor programmer might reason that access 52 * to mutable members doesn't have to be serialized anymore, which would 53 * otherwise imply an acquire fence. Hence conditional acquire fence... 54 */ 55 static inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 56 { 57 #ifdef OSSL_TSAN_BUILD 58 /* 59 * TSAN requires acq_rel as it indicates a false positive error when 60 * the object that contains the refcount is freed otherwise. 61 */ 62 *ret = atomic_fetch_sub_explicit(&refcnt->val, 1, memory_order_acq_rel) - 1; 63 #else 64 *ret = atomic_fetch_sub_explicit(&refcnt->val, 1, memory_order_release) - 1; 65 if (*ret == 0) 66 atomic_thread_fence(memory_order_acquire); 67 #endif 68 return 1; 69 } 70 71 static inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 72 { 73 *ret = atomic_load_explicit(&refcnt->val, memory_order_acquire); 74 return 1; 75 } 76 77 #elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0 78 79 #define HAVE_ATOMICS 1 80 81 typedef struct { 82 int val; 83 } CRYPTO_REF_COUNT; 84 85 static __inline__ int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 86 { 87 *ret = __atomic_fetch_add(&refcnt->val, 1, __ATOMIC_RELAXED) + 1; 88 return 1; 89 } 90 91 static __inline__ int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 92 { 93 *ret = __atomic_fetch_sub(&refcnt->val, 1, __ATOMIC_RELEASE) - 1; 94 if (*ret == 0) 95 __atomic_thread_fence(__ATOMIC_ACQUIRE); 96 return 1; 97 } 98 99 static __inline__ int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 100 { 101 *ret = __atomic_load_n(&refcnt->val, __ATOMIC_RELAXED); 102 return 1; 103 } 104 105 #elif defined(__ICL) && defined(_WIN32) 106 #include <intrin.h> 107 #define HAVE_ATOMICS 1 108 109 typedef struct { 110 volatile long val; 111 } CRYPTO_REF_COUNT; 112 113 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 114 { 115 *ret = _InterlockedExchangeAdd(&refcnt->val, 1) + 1; 116 return 1; 117 } 118 119 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 120 { 121 *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1; 122 return 1; 123 } 124 125 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 126 { 127 *ret = _InterlockedExchangeAdd(&refcnt->val, 0); 128 return 1; 129 } 130 131 #elif defined(_MSC_VER) && _MSC_VER >= 1200 132 133 #define HAVE_ATOMICS 1 134 135 typedef struct { 136 volatile long val; 137 } CRYPTO_REF_COUNT; 138 139 #if (defined(_M_ARM) && _M_ARM >= 7 && !defined(_WIN32_WCE)) || defined(_M_ARM64) 140 #include <intrin.h> 141 #if defined(_M_ARM64) && !defined(_ARM_BARRIER_ISH) 142 #define _ARM_BARRIER_ISH _ARM64_BARRIER_ISH 143 #endif 144 145 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 146 { 147 *ret = _InterlockedExchangeAdd_nf(&refcnt->val, 1) + 1; 148 return 1; 149 } 150 151 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 152 { 153 *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1; 154 return 1; 155 } 156 157 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 158 { 159 *ret = _InterlockedExchangeAdd_acq(&refcnt->val, 0); 160 return 1; 161 } 162 163 #else 164 #if !defined(_WIN32_WCE) 165 #pragma intrinsic(_InterlockedExchangeAdd) 166 #else 167 #if _WIN32_WCE >= 0x600 168 extern long __cdecl _InterlockedExchangeAdd(long volatile *, long); 169 #else 170 /* under Windows CE we still have old-style Interlocked* functions */ 171 extern long __cdecl InterlockedExchangeAdd(long volatile *, long); 172 #define _InterlockedExchangeAdd InterlockedExchangeAdd 173 #endif 174 #endif 175 176 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 177 { 178 *ret = _InterlockedExchangeAdd(&refcnt->val, 1) + 1; 179 return 1; 180 } 181 182 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 183 { 184 *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1; 185 return 1; 186 } 187 188 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 189 { 190 *ret = _InterlockedExchangeAdd(&refcnt->val, 0); 191 return 1; 192 } 193 194 #endif 195 196 #endif 197 #endif /* !OPENSSL_DEV_NO_ATOMICS */ 198 199 /* 200 * All the refcounting implementations above define HAVE_ATOMICS, so if it's 201 * still undefined here (such as when OPENSSL_DEV_NO_ATOMICS is defined), it 202 * means we need to implement a fallback. This fallback uses locks. 203 */ 204 #ifndef HAVE_ATOMICS 205 206 typedef struct { 207 int val; 208 #ifdef OPENSSL_THREADS 209 CRYPTO_RWLOCK *lock; 210 #endif 211 } CRYPTO_REF_COUNT; 212 213 #ifdef OPENSSL_THREADS 214 215 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, 216 int *ret) 217 { 218 return CRYPTO_atomic_add(&refcnt->val, 1, ret, refcnt->lock); 219 } 220 221 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, 222 int *ret) 223 { 224 return CRYPTO_atomic_add(&refcnt->val, -1, ret, refcnt->lock); 225 } 226 227 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, 228 int *ret) 229 { 230 return CRYPTO_atomic_load_int(&refcnt->val, ret, refcnt->lock); 231 } 232 233 #define CRYPTO_NEW_FREE_DEFINED 1 234 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n) 235 { 236 refcnt->val = n; 237 refcnt->lock = CRYPTO_THREAD_lock_new(); 238 if (refcnt->lock == NULL) { 239 ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); 240 return 0; 241 } 242 return 1; 243 } 244 245 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt) 246 { 247 if (refcnt != NULL) 248 CRYPTO_THREAD_lock_free(refcnt->lock); 249 } 250 251 #else /* OPENSSL_THREADS */ 252 253 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, 254 int *ret) 255 { 256 refcnt->val++; 257 *ret = refcnt->val; 258 return 1; 259 } 260 261 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, 262 int *ret) 263 { 264 refcnt->val--; 265 *ret = refcnt->val; 266 return 1; 267 } 268 269 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, 270 int *ret) 271 { 272 *ret = refcnt->val; 273 return 1; 274 } 275 276 #endif /* OPENSSL_THREADS */ 277 #endif 278 279 #ifndef CRYPTO_NEW_FREE_DEFINED 280 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n) 281 { 282 refcnt->val = n; 283 return 1; 284 } 285 286 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt) 287 { 288 } 289 #endif /* CRYPTO_NEW_FREE_DEFINED */ 290 #undef CRYPTO_NEW_FREE_DEFINED 291 292 #if !defined(NDEBUG) && !defined(OPENSSL_NO_STDIO) 293 #define REF_ASSERT_ISNT(test) \ 294 (void)((test) ? (OPENSSL_die("refcount error", __FILE__, __LINE__), 1) : 0) 295 #else 296 #define REF_ASSERT_ISNT(i) 297 #endif 298 299 #define REF_PRINT_EX(text, count, object) \ 300 OSSL_TRACE3(REF_COUNT, "%p:%4d:%s\n", (object), (count), (text)); 301 #define REF_PRINT_COUNT(text, val, object) \ 302 REF_PRINT_EX(text, val, (void *)object) 303 304 #endif 305