1 /* 2 * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License 2.0 (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 #ifndef OSSL_INTERNAL_REFCOUNT_H 10 # define OSSL_INTERNAL_REFCOUNT_H 11 # pragma once 12 13 # include <openssl/e_os2.h> 14 # include <openssl/trace.h> 15 # include <openssl/err.h> 16 17 # if defined(OPENSSL_THREADS) && !defined(OPENSSL_DEV_NO_ATOMICS) 18 # if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ 19 && !defined(__STDC_NO_ATOMICS__) 20 # include <stdatomic.h> 21 # define HAVE_C11_ATOMICS 22 # endif 23 24 # if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \ 25 && ATOMIC_INT_LOCK_FREE > 0 26 27 # define HAVE_ATOMICS 1 28 29 # if defined(__has_feature) 30 # if __has_feature(thread_sanitizer) 31 # define OSSL_TSAN_BUILD 32 # endif 33 # endif 34 35 typedef struct { 36 _Atomic int val; 37 } CRYPTO_REF_COUNT; 38 39 static inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 40 { 41 *ret = atomic_fetch_add_explicit(&refcnt->val, 1, memory_order_relaxed) + 1; 42 return 1; 43 } 44 45 /* 46 * Changes to shared structure other than reference counter have to be 47 * serialized. And any kind of serialization implies a release fence. This 48 * means that by the time reference counter is decremented all other 49 * changes are visible on all processors. Hence decrement itself can be 50 * relaxed. In case it hits zero, object will be destructed. Since it's 51 * last use of the object, destructor programmer might reason that access 52 * to mutable members doesn't have to be serialized anymore, which would 53 * otherwise imply an acquire fence. Hence conditional acquire fence... 54 */ 55 static inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 56 { 57 # ifdef OSSL_TSAN_BUILD 58 /* 59 * TSAN requires acq_rel as it indicates a false positive error when 60 * the object that contains the refcount is freed otherwise. 61 */ 62 *ret = atomic_fetch_sub_explicit(&refcnt->val, 1, memory_order_acq_rel) - 1; 63 # else 64 *ret = atomic_fetch_sub_explicit(&refcnt->val, 1, memory_order_release) - 1; 65 if (*ret == 0) 66 atomic_thread_fence(memory_order_acquire); 67 # endif 68 return 1; 69 } 70 71 static inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 72 { 73 *ret = atomic_load_explicit(&refcnt->val, memory_order_acquire); 74 return 1; 75 } 76 77 # elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0 78 79 # define HAVE_ATOMICS 1 80 81 typedef struct { 82 int val; 83 } CRYPTO_REF_COUNT; 84 85 static __inline__ int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 86 { 87 *ret = __atomic_fetch_add(&refcnt->val, 1, __ATOMIC_RELAXED) + 1; 88 return 1; 89 } 90 91 static __inline__ int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 92 { 93 *ret = __atomic_fetch_sub(&refcnt->val, 1, __ATOMIC_RELEASE) - 1; 94 if (*ret == 0) 95 __atomic_thread_fence(__ATOMIC_ACQUIRE); 96 return 1; 97 } 98 99 static __inline__ int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 100 { 101 *ret = __atomic_load_n(&refcnt->val, __ATOMIC_RELAXED); 102 return 1; 103 } 104 105 # elif defined(__ICL) && defined(_WIN32) 106 # define HAVE_ATOMICS 1 107 108 typedef struct { 109 volatile int val; 110 } CRYPTO_REF_COUNT; 111 112 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 113 { 114 *ret = _InterlockedExchangeAdd((void *)&refcnt->val, 1) + 1; 115 return 1; 116 } 117 118 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 119 { 120 *ret = _InterlockedExchangeAdd((void *)&refcnt->val, -1) - 1; 121 return 1; 122 } 123 124 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 125 { 126 *ret = _InterlockedExchangeAdd((void *)&refcnt->val, 0); 127 return 1; 128 } 129 130 # elif defined(_MSC_VER) && _MSC_VER>=1200 131 132 # define HAVE_ATOMICS 1 133 134 typedef struct { 135 volatile int val; 136 } CRYPTO_REF_COUNT; 137 138 # if (defined(_M_ARM) && _M_ARM>=7 && !defined(_WIN32_WCE)) || defined(_M_ARM64) 139 # include <intrin.h> 140 # if defined(_M_ARM64) && !defined(_ARM_BARRIER_ISH) 141 # define _ARM_BARRIER_ISH _ARM64_BARRIER_ISH 142 # endif 143 144 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 145 { 146 *ret = _InterlockedExchangeAdd_nf(&refcnt->val, 1) + 1; 147 return 1; 148 } 149 150 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 151 { 152 *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1; 153 return 1; 154 } 155 156 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 157 { 158 *ret = _InterlockedExchangeAdd_acq((void *)&refcnt->val, 0); 159 return 1; 160 } 161 162 # else 163 # if !defined(_WIN32_WCE) 164 # pragma intrinsic(_InterlockedExchangeAdd) 165 # else 166 # if _WIN32_WCE >= 0x600 167 extern long __cdecl _InterlockedExchangeAdd(long volatile*, long); 168 # else 169 /* under Windows CE we still have old-style Interlocked* functions */ 170 extern long __cdecl InterlockedExchangeAdd(long volatile*, long); 171 # define _InterlockedExchangeAdd InterlockedExchangeAdd 172 # endif 173 # endif 174 175 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 176 { 177 *ret = _InterlockedExchangeAdd(&refcnt->val, 1) + 1; 178 return 1; 179 } 180 181 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 182 { 183 *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1; 184 return 1; 185 } 186 187 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) 188 { 189 *ret = _InterlockedExchangeAdd(&refcnt->val, 0); 190 return 1; 191 } 192 193 # endif 194 195 # endif 196 # endif /* !OPENSSL_DEV_NO_ATOMICS */ 197 198 /* 199 * All the refcounting implementations above define HAVE_ATOMICS, so if it's 200 * still undefined here (such as when OPENSSL_DEV_NO_ATOMICS is defined), it 201 * means we need to implement a fallback. This fallback uses locks. 202 */ 203 # ifndef HAVE_ATOMICS 204 205 typedef struct { 206 int val; 207 # ifdef OPENSSL_THREADS 208 CRYPTO_RWLOCK *lock; 209 # endif 210 } CRYPTO_REF_COUNT; 211 212 # ifdef OPENSSL_THREADS 213 214 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, 215 int *ret) 216 { 217 return CRYPTO_atomic_add(&refcnt->val, 1, ret, refcnt->lock); 218 } 219 220 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, 221 int *ret) 222 { 223 return CRYPTO_atomic_add(&refcnt->val, -1, ret, refcnt->lock); 224 } 225 226 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, 227 int *ret) 228 { 229 return CRYPTO_atomic_load_int(&refcnt->val, ret, refcnt->lock); 230 } 231 232 # define CRYPTO_NEW_FREE_DEFINED 1 233 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n) 234 { 235 refcnt->val = n; 236 refcnt->lock = CRYPTO_THREAD_lock_new(); 237 if (refcnt->lock == NULL) { 238 ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); 239 return 0; 240 } 241 return 1; 242 } 243 244 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt) \ 245 { 246 if (refcnt != NULL) 247 CRYPTO_THREAD_lock_free(refcnt->lock); 248 } 249 250 # else /* OPENSSL_THREADS */ 251 252 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, 253 int *ret) 254 { 255 refcnt->val++; 256 *ret = refcnt->val; 257 return 1; 258 } 259 260 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, 261 int *ret) 262 { 263 refcnt->val--; 264 *ret = refcnt->val; 265 return 1; 266 } 267 268 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, 269 int *ret) 270 { 271 *ret = refcnt->val; 272 return 1; 273 } 274 275 # endif /* OPENSSL_THREADS */ 276 # endif 277 278 # ifndef CRYPTO_NEW_FREE_DEFINED 279 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n) 280 { 281 refcnt->val = n; 282 return 1; 283 } 284 285 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt) \ 286 { 287 } 288 # endif /* CRYPTO_NEW_FREE_DEFINED */ 289 #undef CRYPTO_NEW_FREE_DEFINED 290 291 # if !defined(NDEBUG) && !defined(OPENSSL_NO_STDIO) 292 # define REF_ASSERT_ISNT(test) \ 293 (void)((test) ? (OPENSSL_die("refcount error", __FILE__, __LINE__), 1) : 0) 294 # else 295 # define REF_ASSERT_ISNT(i) 296 # endif 297 298 # define REF_PRINT_EX(text, count, object) \ 299 OSSL_TRACE3(REF_COUNT, "%p:%4d:%s\n", (object), (count), (text)); 300 # define REF_PRINT_COUNT(text, val, object) \ 301 REF_PRINT_EX(text, val, (void *)object) 302 303 #endif 304