1 /*===---- immintrin.h - Intel intrinsics -----------------------------------=== 2 * 3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 * See https://llvm.org/LICENSE.txt for license information. 5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 * 7 *===-----------------------------------------------------------------------=== 8 */ 9 10 #ifndef __IMMINTRIN_H 11 #define __IMMINTRIN_H 12 13 #if !defined(__i386__) && !defined(__x86_64__) 14 #error "This header is only meant to be used on x86 and x64 architecture" 15 #endif 16 17 #include <x86gprintrin.h> 18 19 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 20 defined(__MMX__) 21 #include <mmintrin.h> 22 #endif 23 24 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 25 defined(__SSE__) 26 #include <xmmintrin.h> 27 #endif 28 29 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 30 defined(__SSE2__) 31 #include <emmintrin.h> 32 #endif 33 34 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 35 defined(__SSE3__) 36 #include <pmmintrin.h> 37 #endif 38 39 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 40 defined(__SSSE3__) 41 #include <tmmintrin.h> 42 #endif 43 44 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 45 (defined(__SSE4_2__) || defined(__SSE4_1__)) 46 #include <smmintrin.h> 47 #endif 48 49 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 50 (defined(__AES__) || defined(__PCLMUL__)) 51 #include <wmmintrin.h> 52 #endif 53 54 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 55 defined(__CLFLUSHOPT__) 56 #include <clflushoptintrin.h> 57 #endif 58 59 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 60 defined(__CLWB__) 61 #include <clwbintrin.h> 62 #endif 63 64 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 65 defined(__AVX__) 66 #include <avxintrin.h> 67 #endif 68 69 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 70 defined(__AVX2__) 71 #include <avx2intrin.h> 72 #endif 73 74 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 75 defined(__F16C__) 76 #include <f16cintrin.h> 77 #endif 78 79 /* No feature check desired due to internal checks */ 80 #include <bmiintrin.h> 81 82 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 83 defined(__BMI2__) 84 #include <bmi2intrin.h> 85 #endif 86 87 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 88 defined(__LZCNT__) 89 #include <lzcntintrin.h> 90 #endif 91 92 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 93 defined(__POPCNT__) 94 #include <popcntintrin.h> 95 #endif 96 97 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 98 defined(__FMA__) 99 #include <fmaintrin.h> 100 #endif 101 102 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 103 defined(__AVX512F__) 104 #include <avx512fintrin.h> 105 #endif 106 107 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 108 defined(__AVX512VL__) 109 #include <avx512vlintrin.h> 110 #endif 111 112 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 113 defined(__AVX512BW__) 114 #include <avx512bwintrin.h> 115 #endif 116 117 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 118 defined(__AVX512BITALG__) 119 #include <avx512bitalgintrin.h> 120 #endif 121 122 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 123 defined(__AVX512CD__) 124 #include <avx512cdintrin.h> 125 #endif 126 127 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 128 defined(__AVX512VPOPCNTDQ__) 129 #include <avx512vpopcntdqintrin.h> 130 #endif 131 132 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 133 (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__)) 134 #include <avx512vpopcntdqvlintrin.h> 135 #endif 136 137 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 138 defined(__AVX512VNNI__) 139 #include <avx512vnniintrin.h> 140 #endif 141 142 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 143 (defined(__AVX512VL__) && defined(__AVX512VNNI__)) 144 #include <avx512vlvnniintrin.h> 145 #endif 146 147 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 148 defined(__AVXVNNI__) 149 #include <avxvnniintrin.h> 150 #endif 151 152 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 153 defined(__AVX512DQ__) 154 #include <avx512dqintrin.h> 155 #endif 156 157 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 158 (defined(__AVX512VL__) && defined(__AVX512BITALG__)) 159 #include <avx512vlbitalgintrin.h> 160 #endif 161 162 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 163 (defined(__AVX512VL__) && defined(__AVX512BW__)) 164 #include <avx512vlbwintrin.h> 165 #endif 166 167 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 168 (defined(__AVX512VL__) && defined(__AVX512CD__)) 169 #include <avx512vlcdintrin.h> 170 #endif 171 172 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 173 (defined(__AVX512VL__) && defined(__AVX512DQ__)) 174 #include <avx512vldqintrin.h> 175 #endif 176 177 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 178 defined(__AVX512ER__) 179 #include <avx512erintrin.h> 180 #endif 181 182 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 183 defined(__AVX512IFMA__) 184 #include <avx512ifmaintrin.h> 185 #endif 186 187 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 188 (defined(__AVX512IFMA__) && defined(__AVX512VL__)) 189 #include <avx512ifmavlintrin.h> 190 #endif 191 192 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 193 defined(__AVX512VBMI__) 194 #include <avx512vbmiintrin.h> 195 #endif 196 197 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 198 (defined(__AVX512VBMI__) && defined(__AVX512VL__)) 199 #include <avx512vbmivlintrin.h> 200 #endif 201 202 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 203 defined(__AVX512VBMI2__) 204 #include <avx512vbmi2intrin.h> 205 #endif 206 207 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 208 (defined(__AVX512VBMI2__) && defined(__AVX512VL__)) 209 #include <avx512vlvbmi2intrin.h> 210 #endif 211 212 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 213 defined(__AVX512PF__) 214 #include <avx512pfintrin.h> 215 #endif 216 217 /* 218 * FIXME: _Float16 type is legal only when HW support float16 operation. 219 * We use __AVX512FP16__ to identify if float16 is supported or not, so 220 * when float16 is not supported, the related header is not included. 221 * 222 */ 223 #if defined(__AVX512FP16__) 224 #include <avx512fp16intrin.h> 225 #endif 226 227 #if defined(__AVX512FP16__) && defined(__AVX512VL__) 228 #include <avx512vlfp16intrin.h> 229 #endif 230 231 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 232 defined(__AVX512BF16__) 233 #include <avx512bf16intrin.h> 234 #endif 235 236 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 237 (defined(__AVX512VL__) && defined(__AVX512BF16__)) 238 #include <avx512vlbf16intrin.h> 239 #endif 240 241 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 242 defined(__PKU__) 243 #include <pkuintrin.h> 244 #endif 245 246 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 247 defined(__VPCLMULQDQ__) 248 #include <vpclmulqdqintrin.h> 249 #endif 250 251 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 252 defined(__VAES__) 253 #include <vaesintrin.h> 254 #endif 255 256 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 257 defined(__GFNI__) 258 #include <gfniintrin.h> 259 #endif 260 261 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 262 defined(__RDPID__) 263 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103). 264 /// 265 /// \headerfile <immintrin.h> 266 /// 267 /// This intrinsic corresponds to the <c> RDPID </c> instruction. 268 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) 269 _rdpid_u32(void) { 270 return __builtin_ia32_rdpid(); 271 } 272 #endif // __RDPID__ 273 274 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 275 defined(__RDRND__) 276 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 277 _rdrand16_step(unsigned short *__p) 278 { 279 return (int)__builtin_ia32_rdrand16_step(__p); 280 } 281 282 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 283 _rdrand32_step(unsigned int *__p) 284 { 285 return (int)__builtin_ia32_rdrand32_step(__p); 286 } 287 288 #ifdef __x86_64__ 289 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 290 _rdrand64_step(unsigned long long *__p) 291 { 292 return (int)__builtin_ia32_rdrand64_step(__p); 293 } 294 #endif 295 #endif /* __RDRND__ */ 296 297 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 298 defined(__FSGSBASE__) 299 #ifdef __x86_64__ 300 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 301 _readfsbase_u32(void) 302 { 303 return __builtin_ia32_rdfsbase32(); 304 } 305 306 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 307 _readfsbase_u64(void) 308 { 309 return __builtin_ia32_rdfsbase64(); 310 } 311 312 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 313 _readgsbase_u32(void) 314 { 315 return __builtin_ia32_rdgsbase32(); 316 } 317 318 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 319 _readgsbase_u64(void) 320 { 321 return __builtin_ia32_rdgsbase64(); 322 } 323 324 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 325 _writefsbase_u32(unsigned int __V) 326 { 327 __builtin_ia32_wrfsbase32(__V); 328 } 329 330 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 331 _writefsbase_u64(unsigned long long __V) 332 { 333 __builtin_ia32_wrfsbase64(__V); 334 } 335 336 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 337 _writegsbase_u32(unsigned int __V) 338 { 339 __builtin_ia32_wrgsbase32(__V); 340 } 341 342 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 343 _writegsbase_u64(unsigned long long __V) 344 { 345 __builtin_ia32_wrgsbase64(__V); 346 } 347 348 #endif 349 #endif /* __FSGSBASE__ */ 350 351 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 352 defined(__MOVBE__) 353 354 /* The structs used below are to force the load/store to be unaligned. This 355 * is accomplished with the __packed__ attribute. The __may_alias__ prevents 356 * tbaa metadata from being generated based on the struct and the type of the 357 * field inside of it. 358 */ 359 360 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 361 _loadbe_i16(void const * __P) { 362 struct __loadu_i16 { 363 unsigned short __v; 364 } __attribute__((__packed__, __may_alias__)); 365 return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v); 366 } 367 368 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 369 _storebe_i16(void * __P, short __D) { 370 struct __storeu_i16 { 371 unsigned short __v; 372 } __attribute__((__packed__, __may_alias__)); 373 ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D); 374 } 375 376 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 377 _loadbe_i32(void const * __P) { 378 struct __loadu_i32 { 379 unsigned int __v; 380 } __attribute__((__packed__, __may_alias__)); 381 return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v); 382 } 383 384 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 385 _storebe_i32(void * __P, int __D) { 386 struct __storeu_i32 { 387 unsigned int __v; 388 } __attribute__((__packed__, __may_alias__)); 389 ((struct __storeu_i32*)__P)->__v = __builtin_bswap32((unsigned int)__D); 390 } 391 392 #ifdef __x86_64__ 393 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 394 _loadbe_i64(void const * __P) { 395 struct __loadu_i64 { 396 unsigned long long __v; 397 } __attribute__((__packed__, __may_alias__)); 398 return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v); 399 } 400 401 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 402 _storebe_i64(void * __P, long long __D) { 403 struct __storeu_i64 { 404 unsigned long long __v; 405 } __attribute__((__packed__, __may_alias__)); 406 ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D); 407 } 408 #endif 409 #endif /* __MOVBE */ 410 411 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 412 defined(__RTM__) 413 #include <rtmintrin.h> 414 #include <xtestintrin.h> 415 #endif 416 417 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 418 defined(__SHA__) 419 #include <shaintrin.h> 420 #endif 421 422 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 423 defined(__FXSR__) 424 #include <fxsrintrin.h> 425 #endif 426 427 /* No feature check desired due to internal MSC_VER checks */ 428 #include <xsaveintrin.h> 429 430 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 431 defined(__XSAVEOPT__) 432 #include <xsaveoptintrin.h> 433 #endif 434 435 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 436 defined(__XSAVEC__) 437 #include <xsavecintrin.h> 438 #endif 439 440 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 441 defined(__XSAVES__) 442 #include <xsavesintrin.h> 443 #endif 444 445 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 446 defined(__SHSTK__) 447 #include <cetintrin.h> 448 #endif 449 450 /* Some intrinsics inside adxintrin.h are available only on processors with ADX, 451 * whereas others are also available at all times. */ 452 #include <adxintrin.h> 453 454 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 455 defined(__RDSEED__) 456 #include <rdseedintrin.h> 457 #endif 458 459 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 460 defined(__WBNOINVD__) 461 #include <wbnoinvdintrin.h> 462 #endif 463 464 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 465 defined(__CLDEMOTE__) 466 #include <cldemoteintrin.h> 467 #endif 468 469 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 470 defined(__WAITPKG__) 471 #include <waitpkgintrin.h> 472 #endif 473 474 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 475 defined(__MOVDIRI__) || defined(__MOVDIR64B__) 476 #include <movdirintrin.h> 477 #endif 478 479 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 480 defined(__PCONFIG__) 481 #include <pconfigintrin.h> 482 #endif 483 484 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 485 defined(__SGX__) 486 #include <sgxintrin.h> 487 #endif 488 489 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 490 defined(__PTWRITE__) 491 #include <ptwriteintrin.h> 492 #endif 493 494 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 495 defined(__INVPCID__) 496 #include <invpcidintrin.h> 497 #endif 498 499 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 500 defined(__KL__) || defined(__WIDEKL__) 501 #include <keylockerintrin.h> 502 #endif 503 504 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 505 defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__) 506 #include <amxintrin.h> 507 #endif 508 509 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 510 defined(__AVX512VP2INTERSECT__) 511 #include <avx512vp2intersectintrin.h> 512 #endif 513 514 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 515 (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__)) 516 #include <avx512vlvp2intersectintrin.h> 517 #endif 518 519 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 520 defined(__ENQCMD__) 521 #include <enqcmdintrin.h> 522 #endif 523 524 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 525 defined(__SERIALIZE__) 526 #include <serializeintrin.h> 527 #endif 528 529 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 530 defined(__TSXLDTRK__) 531 #include <tsxldtrkintrin.h> 532 #endif 533 534 #if defined(_MSC_VER) && __has_extension(gnu_asm) 535 /* Define the default attributes for these intrinsics */ 536 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 537 #ifdef __cplusplus 538 extern "C" { 539 #endif 540 /*----------------------------------------------------------------------------*\ 541 |* Interlocked Exchange HLE 542 \*----------------------------------------------------------------------------*/ 543 #if defined(__i386__) || defined(__x86_64__) 544 static __inline__ long __DEFAULT_FN_ATTRS 545 _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) { 546 __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}" 547 : "+r" (_Value), "+m" (*_Target) :: "memory"); 548 return _Value; 549 } 550 static __inline__ long __DEFAULT_FN_ATTRS 551 _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) { 552 __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}" 553 : "+r" (_Value), "+m" (*_Target) :: "memory"); 554 return _Value; 555 } 556 #endif 557 #if defined(__x86_64__) 558 static __inline__ __int64 __DEFAULT_FN_ATTRS 559 _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) { 560 __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}" 561 : "+r" (_Value), "+m" (*_Target) :: "memory"); 562 return _Value; 563 } 564 static __inline__ __int64 __DEFAULT_FN_ATTRS 565 _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) { 566 __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}" 567 : "+r" (_Value), "+m" (*_Target) :: "memory"); 568 return _Value; 569 } 570 #endif 571 /*----------------------------------------------------------------------------*\ 572 |* Interlocked Compare Exchange HLE 573 \*----------------------------------------------------------------------------*/ 574 #if defined(__i386__) || defined(__x86_64__) 575 static __inline__ long __DEFAULT_FN_ATTRS 576 _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination, 577 long _Exchange, long _Comparand) { 578 __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}" 579 : "+a" (_Comparand), "+m" (*_Destination) 580 : "r" (_Exchange) : "memory"); 581 return _Comparand; 582 } 583 static __inline__ long __DEFAULT_FN_ATTRS 584 _InterlockedCompareExchange_HLERelease(long volatile *_Destination, 585 long _Exchange, long _Comparand) { 586 __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}" 587 : "+a" (_Comparand), "+m" (*_Destination) 588 : "r" (_Exchange) : "memory"); 589 return _Comparand; 590 } 591 #endif 592 #if defined(__x86_64__) 593 static __inline__ __int64 __DEFAULT_FN_ATTRS 594 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination, 595 __int64 _Exchange, __int64 _Comparand) { 596 __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}" 597 : "+a" (_Comparand), "+m" (*_Destination) 598 : "r" (_Exchange) : "memory"); 599 return _Comparand; 600 } 601 static __inline__ __int64 __DEFAULT_FN_ATTRS 602 _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination, 603 __int64 _Exchange, __int64 _Comparand) { 604 __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}" 605 : "+a" (_Comparand), "+m" (*_Destination) 606 : "r" (_Exchange) : "memory"); 607 return _Comparand; 608 } 609 #endif 610 #ifdef __cplusplus 611 } 612 #endif 613 614 #undef __DEFAULT_FN_ATTRS 615 616 #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */ 617 618 #endif /* __IMMINTRIN_H */ 619