1*0fca6ea1SDimitry Andric /* ===-------- intrin.h ---------------------------------------------------=== 2*0fca6ea1SDimitry Andric * 3*0fca6ea1SDimitry Andric * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4*0fca6ea1SDimitry Andric * See https://llvm.org/LICENSE.txt for license information. 5*0fca6ea1SDimitry Andric * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6*0fca6ea1SDimitry Andric * 7*0fca6ea1SDimitry Andric *===-----------------------------------------------------------------------=== 8*0fca6ea1SDimitry Andric */ 9*0fca6ea1SDimitry Andric 10*0fca6ea1SDimitry Andric /* Only include this if we're compiling for the windows platform. */ 11*0fca6ea1SDimitry Andric #ifndef _MSC_VER 12*0fca6ea1SDimitry Andric #include_next <intrin0.h> 13*0fca6ea1SDimitry Andric #else 14*0fca6ea1SDimitry Andric 15*0fca6ea1SDimitry Andric #ifndef __INTRIN0_H 16*0fca6ea1SDimitry Andric #define __INTRIN0_H 17*0fca6ea1SDimitry Andric 18*0fca6ea1SDimitry Andric #if defined(__x86_64__) && !defined(__arm64ec__) 19*0fca6ea1SDimitry Andric #include <adcintrin.h> 20*0fca6ea1SDimitry Andric #endif 21*0fca6ea1SDimitry Andric 22*0fca6ea1SDimitry Andric #ifdef __cplusplus 23*0fca6ea1SDimitry Andric extern "C" { 24*0fca6ea1SDimitry Andric #endif 25*0fca6ea1SDimitry Andric 26*0fca6ea1SDimitry Andric unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); 27*0fca6ea1SDimitry Andric unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); 28*0fca6ea1SDimitry Andric void _ReadWriteBarrier(void); 29*0fca6ea1SDimitry Andric 30*0fca6ea1SDimitry Andric #if defined(__aarch64__) || defined(__arm64ec__) 31*0fca6ea1SDimitry Andric unsigned int _CountLeadingZeros(unsigned long); 32*0fca6ea1SDimitry Andric unsigned int _CountLeadingZeros64(unsigned _int64); 33*0fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128_acq(__int64 volatile *_Destination, 34*0fca6ea1SDimitry Andric __int64 _ExchangeHigh, 35*0fca6ea1SDimitry Andric __int64 _ExchangeLow, 36*0fca6ea1SDimitry Andric __int64 *_ComparandResult); 37*0fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128_nf(__int64 volatile *_Destination, 38*0fca6ea1SDimitry Andric __int64 _ExchangeHigh, 39*0fca6ea1SDimitry Andric __int64 _ExchangeLow, 40*0fca6ea1SDimitry Andric __int64 *_ComparandResult); 41*0fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128_rel(__int64 volatile *_Destination, 42*0fca6ea1SDimitry Andric __int64 _ExchangeHigh, 43*0fca6ea1SDimitry Andric __int64 _ExchangeLow, 44*0fca6ea1SDimitry Andric __int64 *_ComparandResult); 45*0fca6ea1SDimitry Andric #endif 46*0fca6ea1SDimitry Andric 47*0fca6ea1SDimitry Andric #ifdef __x86_64__ && !defined(__arm64ec__) 48*0fca6ea1SDimitry Andric unsigned __int64 _umul128(unsigned __int64, unsigned __int64, 49*0fca6ea1SDimitry Andric unsigned __int64 *); 50*0fca6ea1SDimitry Andric unsigned __int64 __shiftleft128(unsigned __int64 _LowPart, 51*0fca6ea1SDimitry Andric unsigned __int64 _HighPart, 52*0fca6ea1SDimitry Andric unsigned char _Shift); 53*0fca6ea1SDimitry Andric unsigned __int64 __shiftright128(unsigned __int64 _LowPart, 54*0fca6ea1SDimitry Andric unsigned __int64 _HighPart, 55*0fca6ea1SDimitry Andric unsigned char _Shift); 56*0fca6ea1SDimitry Andric #endif 57*0fca6ea1SDimitry Andric 58*0fca6ea1SDimitry Andric #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__)) 59*0fca6ea1SDimitry Andric void _mm_pause(void); 60*0fca6ea1SDimitry Andric #endif 61*0fca6ea1SDimitry Andric 62*0fca6ea1SDimitry Andric #if defined(__x86_64__) || defined(__aarch64__) 63*0fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination, 64*0fca6ea1SDimitry Andric __int64 _ExchangeHigh, 65*0fca6ea1SDimitry Andric __int64 _ExchangeLow, 66*0fca6ea1SDimitry Andric __int64 *_ComparandResult); 67*0fca6ea1SDimitry Andric #endif 68*0fca6ea1SDimitry Andric 69*0fca6ea1SDimitry Andric #if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__) 70*0fca6ea1SDimitry Andric unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); 71*0fca6ea1SDimitry Andric unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); 72*0fca6ea1SDimitry Andric #endif 73*0fca6ea1SDimitry Andric 74*0fca6ea1SDimitry Andric #if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ 75*0fca6ea1SDimitry Andric defined(__aarch64__) 76*0fca6ea1SDimitry Andric __int64 _InterlockedDecrement64(__int64 volatile *_Addend); 77*0fca6ea1SDimitry Andric __int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value); 78*0fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value); 79*0fca6ea1SDimitry Andric __int64 _InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value); 80*0fca6ea1SDimitry Andric __int64 _InterlockedIncrement64(__int64 volatile *_Addend); 81*0fca6ea1SDimitry Andric __int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask); 82*0fca6ea1SDimitry Andric __int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask); 83*0fca6ea1SDimitry Andric __int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask); 84*0fca6ea1SDimitry Andric #endif 85*0fca6ea1SDimitry Andric 86*0fca6ea1SDimitry Andric #if defined(__arm__) || defined(__aarch64__) || defined(__arm64ec__) 87*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 88*0fca6ea1SDimitry Andric |* Interlocked Exchange Add 89*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 90*0fca6ea1SDimitry Andric char _InterlockedExchangeAdd8_acq(char volatile *_Addend, char _Value); 91*0fca6ea1SDimitry Andric char _InterlockedExchangeAdd8_nf(char volatile *_Addend, char _Value); 92*0fca6ea1SDimitry Andric char _InterlockedExchangeAdd8_rel(char volatile *_Addend, char _Value); 93*0fca6ea1SDimitry Andric short _InterlockedExchangeAdd16_acq(short volatile *_Addend, short _Value); 94*0fca6ea1SDimitry Andric short _InterlockedExchangeAdd16_nf(short volatile *_Addend, short _Value); 95*0fca6ea1SDimitry Andric short _InterlockedExchangeAdd16_rel(short volatile *_Addend, short _Value); 96*0fca6ea1SDimitry Andric long _InterlockedExchangeAdd_acq(long volatile *_Addend, long _Value); 97*0fca6ea1SDimitry Andric long _InterlockedExchangeAdd_nf(long volatile *_Addend, long _Value); 98*0fca6ea1SDimitry Andric long _InterlockedExchangeAdd_rel(long volatile *_Addend, long _Value); 99*0fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64_acq(__int64 volatile *_Addend, 100*0fca6ea1SDimitry Andric __int64 _Value); 101*0fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64_nf(__int64 volatile *_Addend, __int64 _Value); 102*0fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64_rel(__int64 volatile *_Addend, 103*0fca6ea1SDimitry Andric __int64 _Value); 104*0fca6ea1SDimitry Andric 105*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 106*0fca6ea1SDimitry Andric |* Interlocked Increment 107*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 108*0fca6ea1SDimitry Andric short _InterlockedIncrement16_acq(short volatile *_Value); 109*0fca6ea1SDimitry Andric short _InterlockedIncrement16_nf(short volatile *_Value); 110*0fca6ea1SDimitry Andric short _InterlockedIncrement16_rel(short volatile *_Value); 111*0fca6ea1SDimitry Andric long _InterlockedIncrement_acq(long volatile *_Value); 112*0fca6ea1SDimitry Andric long _InterlockedIncrement_nf(long volatile *_Value); 113*0fca6ea1SDimitry Andric long _InterlockedIncrement_rel(long volatile *_Value); 114*0fca6ea1SDimitry Andric __int64 _InterlockedIncrement64_acq(__int64 volatile *_Value); 115*0fca6ea1SDimitry Andric __int64 _InterlockedIncrement64_nf(__int64 volatile *_Value); 116*0fca6ea1SDimitry Andric __int64 _InterlockedIncrement64_rel(__int64 volatile *_Value); 117*0fca6ea1SDimitry Andric 118*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 119*0fca6ea1SDimitry Andric |* Interlocked Decrement 120*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 121*0fca6ea1SDimitry Andric short _InterlockedDecrement16_acq(short volatile *_Value); 122*0fca6ea1SDimitry Andric short _InterlockedDecrement16_nf(short volatile *_Value); 123*0fca6ea1SDimitry Andric short _InterlockedDecrement16_rel(short volatile *_Value); 124*0fca6ea1SDimitry Andric long _InterlockedDecrement_acq(long volatile *_Value); 125*0fca6ea1SDimitry Andric long _InterlockedDecrement_nf(long volatile *_Value); 126*0fca6ea1SDimitry Andric long _InterlockedDecrement_rel(long volatile *_Value); 127*0fca6ea1SDimitry Andric __int64 _InterlockedDecrement64_acq(__int64 volatile *_Value); 128*0fca6ea1SDimitry Andric __int64 _InterlockedDecrement64_nf(__int64 volatile *_Value); 129*0fca6ea1SDimitry Andric __int64 _InterlockedDecrement64_rel(__int64 volatile *_Value); 130*0fca6ea1SDimitry Andric 131*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 132*0fca6ea1SDimitry Andric |* Interlocked And 133*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 134*0fca6ea1SDimitry Andric char _InterlockedAnd8_acq(char volatile *_Value, char _Mask); 135*0fca6ea1SDimitry Andric char _InterlockedAnd8_nf(char volatile *_Value, char _Mask); 136*0fca6ea1SDimitry Andric char _InterlockedAnd8_rel(char volatile *_Value, char _Mask); 137*0fca6ea1SDimitry Andric short _InterlockedAnd16_acq(short volatile *_Value, short _Mask); 138*0fca6ea1SDimitry Andric short _InterlockedAnd16_nf(short volatile *_Value, short _Mask); 139*0fca6ea1SDimitry Andric short _InterlockedAnd16_rel(short volatile *_Value, short _Mask); 140*0fca6ea1SDimitry Andric long _InterlockedAnd_acq(long volatile *_Value, long _Mask); 141*0fca6ea1SDimitry Andric long _InterlockedAnd_nf(long volatile *_Value, long _Mask); 142*0fca6ea1SDimitry Andric long _InterlockedAnd_rel(long volatile *_Value, long _Mask); 143*0fca6ea1SDimitry Andric __int64 _InterlockedAnd64_acq(__int64 volatile *_Value, __int64 _Mask); 144*0fca6ea1SDimitry Andric __int64 _InterlockedAnd64_nf(__int64 volatile *_Value, __int64 _Mask); 145*0fca6ea1SDimitry Andric __int64 _InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask); 146*0fca6ea1SDimitry Andric 147*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 148*0fca6ea1SDimitry Andric |* Bit Counting and Testing 149*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 150*0fca6ea1SDimitry Andric unsigned char _interlockedbittestandset_acq(long volatile *_BitBase, 151*0fca6ea1SDimitry Andric long _BitPos); 152*0fca6ea1SDimitry Andric unsigned char _interlockedbittestandset_nf(long volatile *_BitBase, 153*0fca6ea1SDimitry Andric long _BitPos); 154*0fca6ea1SDimitry Andric unsigned char _interlockedbittestandset_rel(long volatile *_BitBase, 155*0fca6ea1SDimitry Andric long _BitPos); 156*0fca6ea1SDimitry Andric unsigned char _interlockedbittestandreset_acq(long volatile *_BitBase, 157*0fca6ea1SDimitry Andric long _BitPos); 158*0fca6ea1SDimitry Andric unsigned char _interlockedbittestandreset_nf(long volatile *_BitBase, 159*0fca6ea1SDimitry Andric long _BitPos); 160*0fca6ea1SDimitry Andric unsigned char _interlockedbittestandreset_rel(long volatile *_BitBase, 161*0fca6ea1SDimitry Andric long _BitPos); 162*0fca6ea1SDimitry Andric 163*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 164*0fca6ea1SDimitry Andric |* Interlocked Or 165*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 166*0fca6ea1SDimitry Andric char _InterlockedOr8_acq(char volatile *_Value, char _Mask); 167*0fca6ea1SDimitry Andric char _InterlockedOr8_nf(char volatile *_Value, char _Mask); 168*0fca6ea1SDimitry Andric char _InterlockedOr8_rel(char volatile *_Value, char _Mask); 169*0fca6ea1SDimitry Andric short _InterlockedOr16_acq(short volatile *_Value, short _Mask); 170*0fca6ea1SDimitry Andric short _InterlockedOr16_nf(short volatile *_Value, short _Mask); 171*0fca6ea1SDimitry Andric short _InterlockedOr16_rel(short volatile *_Value, short _Mask); 172*0fca6ea1SDimitry Andric long _InterlockedOr_acq(long volatile *_Value, long _Mask); 173*0fca6ea1SDimitry Andric long _InterlockedOr_nf(long volatile *_Value, long _Mask); 174*0fca6ea1SDimitry Andric long _InterlockedOr_rel(long volatile *_Value, long _Mask); 175*0fca6ea1SDimitry Andric __int64 _InterlockedOr64_acq(__int64 volatile *_Value, __int64 _Mask); 176*0fca6ea1SDimitry Andric __int64 _InterlockedOr64_nf(__int64 volatile *_Value, __int64 _Mask); 177*0fca6ea1SDimitry Andric __int64 _InterlockedOr64_rel(__int64 volatile *_Value, __int64 _Mask); 178*0fca6ea1SDimitry Andric 179*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 180*0fca6ea1SDimitry Andric |* Interlocked Xor 181*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 182*0fca6ea1SDimitry Andric char _InterlockedXor8_acq(char volatile *_Value, char _Mask); 183*0fca6ea1SDimitry Andric char _InterlockedXor8_nf(char volatile *_Value, char _Mask); 184*0fca6ea1SDimitry Andric char _InterlockedXor8_rel(char volatile *_Value, char _Mask); 185*0fca6ea1SDimitry Andric short _InterlockedXor16_acq(short volatile *_Value, short _Mask); 186*0fca6ea1SDimitry Andric short _InterlockedXor16_nf(short volatile *_Value, short _Mask); 187*0fca6ea1SDimitry Andric short _InterlockedXor16_rel(short volatile *_Value, short _Mask); 188*0fca6ea1SDimitry Andric long _InterlockedXor_acq(long volatile *_Value, long _Mask); 189*0fca6ea1SDimitry Andric long _InterlockedXor_nf(long volatile *_Value, long _Mask); 190*0fca6ea1SDimitry Andric long _InterlockedXor_rel(long volatile *_Value, long _Mask); 191*0fca6ea1SDimitry Andric __int64 _InterlockedXor64_acq(__int64 volatile *_Value, __int64 _Mask); 192*0fca6ea1SDimitry Andric __int64 _InterlockedXor64_nf(__int64 volatile *_Value, __int64 _Mask); 193*0fca6ea1SDimitry Andric __int64 _InterlockedXor64_rel(__int64 volatile *_Value, __int64 _Mask); 194*0fca6ea1SDimitry Andric 195*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 196*0fca6ea1SDimitry Andric |* Interlocked Exchange 197*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 198*0fca6ea1SDimitry Andric char _InterlockedExchange8_acq(char volatile *_Target, char _Value); 199*0fca6ea1SDimitry Andric char _InterlockedExchange8_nf(char volatile *_Target, char _Value); 200*0fca6ea1SDimitry Andric char _InterlockedExchange8_rel(char volatile *_Target, char _Value); 201*0fca6ea1SDimitry Andric short _InterlockedExchange16_acq(short volatile *_Target, short _Value); 202*0fca6ea1SDimitry Andric short _InterlockedExchange16_nf(short volatile *_Target, short _Value); 203*0fca6ea1SDimitry Andric short _InterlockedExchange16_rel(short volatile *_Target, short _Value); 204*0fca6ea1SDimitry Andric long _InterlockedExchange_acq(long volatile *_Target, long _Value); 205*0fca6ea1SDimitry Andric long _InterlockedExchange_nf(long volatile *_Target, long _Value); 206*0fca6ea1SDimitry Andric long _InterlockedExchange_rel(long volatile *_Target, long _Value); 207*0fca6ea1SDimitry Andric __int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value); 208*0fca6ea1SDimitry Andric __int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value); 209*0fca6ea1SDimitry Andric __int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value); 210*0fca6ea1SDimitry Andric 211*0fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 212*0fca6ea1SDimitry Andric |* Interlocked Compare Exchange 213*0fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 214*0fca6ea1SDimitry Andric char _InterlockedCompareExchange8_acq(char volatile *_Destination, 215*0fca6ea1SDimitry Andric char _Exchange, char _Comparand); 216*0fca6ea1SDimitry Andric char _InterlockedCompareExchange8_nf(char volatile *_Destination, 217*0fca6ea1SDimitry Andric char _Exchange, char _Comparand); 218*0fca6ea1SDimitry Andric char _InterlockedCompareExchange8_rel(char volatile *_Destination, 219*0fca6ea1SDimitry Andric char _Exchange, char _Comparand); 220*0fca6ea1SDimitry Andric short _InterlockedCompareExchange16_acq(short volatile *_Destination, 221*0fca6ea1SDimitry Andric short _Exchange, short _Comparand); 222*0fca6ea1SDimitry Andric short _InterlockedCompareExchange16_nf(short volatile *_Destination, 223*0fca6ea1SDimitry Andric short _Exchange, short _Comparand); 224*0fca6ea1SDimitry Andric short _InterlockedCompareExchange16_rel(short volatile *_Destination, 225*0fca6ea1SDimitry Andric short _Exchange, short _Comparand); 226*0fca6ea1SDimitry Andric long _InterlockedCompareExchange_acq(long volatile *_Destination, 227*0fca6ea1SDimitry Andric long _Exchange, long _Comparand); 228*0fca6ea1SDimitry Andric long _InterlockedCompareExchange_nf(long volatile *_Destination, long _Exchange, 229*0fca6ea1SDimitry Andric long _Comparand); 230*0fca6ea1SDimitry Andric long _InterlockedCompareExchange_rel(long volatile *_Destination, 231*0fca6ea1SDimitry Andric long _Exchange, long _Comparand); 232*0fca6ea1SDimitry Andric __int64 _InterlockedCompareExchange64_acq(__int64 volatile *_Destination, 233*0fca6ea1SDimitry Andric __int64 _Exchange, 234*0fca6ea1SDimitry Andric __int64 _Comparand); 235*0fca6ea1SDimitry Andric __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination, 236*0fca6ea1SDimitry Andric __int64 _Exchange, __int64 _Comparand); 237*0fca6ea1SDimitry Andric __int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination, 238*0fca6ea1SDimitry Andric __int64 _Exchange, 239*0fca6ea1SDimitry Andric __int64 _Comparand); 240*0fca6ea1SDimitry Andric #endif 241*0fca6ea1SDimitry Andric 242*0fca6ea1SDimitry Andric #ifdef __cplusplus 243*0fca6ea1SDimitry Andric } 244*0fca6ea1SDimitry Andric #endif 245*0fca6ea1SDimitry Andric 246*0fca6ea1SDimitry Andric #endif /* __INTRIN0_H */ 247*0fca6ea1SDimitry Andric #endif /* _MSC_VER */ 248