1 /* 2 * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License 2.0 (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 /* 11 * Contemporary compilers implement lock-free atomic memory access 12 * primitives that facilitate writing "thread-opportunistic" or even real 13 * multi-threading low-overhead code. "Thread-opportunistic" is when 14 * exact result is not required, e.g. some statistics, or execution flow 15 * doesn't have to be unambiguous. Simplest example is lazy "constant" 16 * initialization when one can synchronize on variable itself, e.g. 17 * 18 * if (var == NOT_YET_INITIALIZED) 19 * var = function_returning_same_value(); 20 * 21 * This does work provided that loads and stores are single-instruction 22 * operations (and integer ones are on *all* supported platforms), but 23 * it upsets Thread Sanitizer. Suggested solution is 24 * 25 * if (tsan_load(&var) == NOT_YET_INITIALIZED) 26 * tsan_store(&var, function_returning_same_value()); 27 * 28 * Production machine code would be the same, so one can wonder why 29 * bother. Having Thread Sanitizer accept "thread-opportunistic" code 30 * allows to move on trouble-shooting real bugs. 31 * 32 * Resolving Thread Sanitizer nits was the initial purpose for this module, 33 * but it was later extended with more nuanced primitives that are useful 34 * even in "non-opportunistic" scenarios. Most notably verifying if a shared 35 * structure is fully initialized and bypassing the initialization lock. 36 * It's suggested to view macros defined in this module as "annotations" for 37 * thread-safe lock-free code, "Thread-Safe ANnotations"... 38 * 39 * It's assumed that ATOMIC_{LONG|INT}_LOCK_FREE are assigned same value as 40 * ATOMIC_POINTER_LOCK_FREE. And check for >= 2 ensures that corresponding 41 * code is inlined. It should be noted that statistics counters become 42 * accurate in such case. 43 * 44 * Special note about TSAN_QUALIFIER. It might be undesired to use it in 45 * a shared header. Because whether operation on specific variable or member 46 * is atomic or not might be irrelevant in other modules. In such case one 47 * can use TSAN_QUALIFIER in cast specifically when it has to count. 48 */ 49 50 #ifndef OSSL_INTERNAL_TSAN_ASSIST_H 51 #define OSSL_INTERNAL_TSAN_ASSIST_H 52 #pragma once 53 54 #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ 55 && !defined(__STDC_NO_ATOMICS__) 56 #include <stdatomic.h> 57 58 #if defined(ATOMIC_POINTER_LOCK_FREE) \ 59 && ATOMIC_POINTER_LOCK_FREE >= 2 60 #define TSAN_QUALIFIER _Atomic 61 #define tsan_load(ptr) atomic_load_explicit((ptr), memory_order_relaxed) 62 #define tsan_store(ptr, val) atomic_store_explicit((ptr), (val), memory_order_relaxed) 63 #define tsan_add(ptr, n) atomic_fetch_add_explicit((ptr), (n), memory_order_relaxed) 64 #define tsan_ld_acq(ptr) atomic_load_explicit((ptr), memory_order_acquire) 65 #define tsan_st_rel(ptr, val) atomic_store_explicit((ptr), (val), memory_order_release) 66 #endif 67 68 #elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) 69 70 #if defined(__GCC_ATOMIC_POINTER_LOCK_FREE) \ 71 && __GCC_ATOMIC_POINTER_LOCK_FREE >= 2 72 #define TSAN_QUALIFIER volatile 73 #define tsan_load(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED) 74 #define tsan_store(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELAXED) 75 #define tsan_add(ptr, n) __atomic_fetch_add((ptr), (n), __ATOMIC_RELAXED) 76 #define tsan_ld_acq(ptr) __atomic_load_n((ptr), __ATOMIC_ACQUIRE) 77 #define tsan_st_rel(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE) 78 #endif 79 80 #elif defined(_MSC_VER) && _MSC_VER >= 1200 \ 81 && (defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || defined(_M_ARM64) || (defined(_M_ARM) && _M_ARM >= 7 && !defined(_WIN32_WCE))) 82 /* 83 * There is subtle dependency on /volatile:<iso|ms> command-line option. 84 * "ms" implies same semantic as memory_order_acquire for loads and 85 * memory_order_release for stores, while "iso" - memory_order_relaxed for 86 * either. Real complication is that defaults are different on x86 and ARM. 87 * There is explanation for that, "ms" is backward compatible with earlier 88 * compiler versions, while multi-processor ARM can be viewed as brand new 89 * platform to MSC and its users, and with non-relaxed semantic taking toll 90 * with additional instructions and penalties, it kind of makes sense to 91 * default to "iso"... 92 */ 93 #define TSAN_QUALIFIER volatile 94 #if defined(_M_ARM) || defined(_M_ARM64) 95 #define _InterlockedExchangeAdd _InterlockedExchangeAdd_nf 96 #pragma intrinsic(_InterlockedExchangeAdd_nf) 97 #pragma intrinsic(__iso_volatile_load32, __iso_volatile_store32) 98 #ifdef _WIN64 99 #define _InterlockedExchangeAdd64 _InterlockedExchangeAdd64_nf 100 #pragma intrinsic(_InterlockedExchangeAdd64_nf) 101 #pragma intrinsic(__iso_volatile_load64, __iso_volatile_store64) 102 #define tsan_load(ptr) (sizeof(*(ptr)) == 8 ? __iso_volatile_load64(ptr) \ 103 : __iso_volatile_load32(ptr)) 104 #define tsan_store(ptr, val) (sizeof(*(ptr)) == 8 ? __iso_volatile_store64((ptr), (val)) \ 105 : __iso_volatile_store32((ptr), (val))) 106 #else 107 #define tsan_load(ptr) __iso_volatile_load32(ptr) 108 #define tsan_store(ptr, val) __iso_volatile_store32((ptr), (val)) 109 #endif 110 #else 111 #define tsan_load(ptr) (*(ptr)) 112 #define tsan_store(ptr, val) (*(ptr) = (val)) 113 #endif 114 #pragma intrinsic(_InterlockedExchangeAdd) 115 #ifdef _WIN64 116 #pragma intrinsic(_InterlockedExchangeAdd64) 117 #define tsan_add(ptr, n) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), (n)) \ 118 : _InterlockedExchangeAdd((ptr), (n))) 119 #else 120 #define tsan_add(ptr, n) _InterlockedExchangeAdd((ptr), (n)) 121 #endif 122 #if !defined(_ISO_VOLATILE) 123 #define tsan_ld_acq(ptr) (*(ptr)) 124 #define tsan_st_rel(ptr, val) (*(ptr) = (val)) 125 #endif 126 127 #endif 128 129 #ifndef TSAN_QUALIFIER 130 131 #ifdef OPENSSL_THREADS 132 #define TSAN_QUALIFIER volatile 133 #define TSAN_REQUIRES_LOCKING 134 #else /* OPENSSL_THREADS */ 135 #define TSAN_QUALIFIER 136 #endif /* OPENSSL_THREADS */ 137 138 #define tsan_load(ptr) (*(ptr)) 139 #define tsan_store(ptr, val) (*(ptr) = (val)) 140 #define tsan_add(ptr, n) (*(ptr) += (n)) 141 /* 142 * Lack of tsan_ld_acq and tsan_ld_rel means that compiler support is not 143 * sophisticated enough to support them. Code that relies on them should be 144 * protected with #ifdef tsan_ld_acq with locked fallback. 145 */ 146 147 #endif 148 149 #define tsan_counter(ptr) tsan_add((ptr), 1) 150 #define tsan_decr(ptr) tsan_add((ptr), -1) 151 152 #endif 153