1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H 10 #define _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H 11 12 #include <__cxx03/__algorithm/min.h> 13 #include <__cxx03/__bit/countl.h> 14 #include <__cxx03/__bit/countr.h> 15 #include <__cxx03/__config> 16 #include <__cxx03/__type_traits/is_arithmetic.h> 17 #include <__cxx03/__type_traits/is_same.h> 18 #include <__cxx03/__utility/integer_sequence.h> 19 #include <__cxx03/cstddef> 20 #include <__cxx03/cstdint> 21 22 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 23 # pragma GCC system_header 24 #endif 25 26 _LIBCPP_PUSH_MACROS 27 #include <__cxx03/__undef_macros> 28 29 // TODO: Find out how altivec changes things and allow vectorizations there too. 30 #define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0 31 32 #if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__) 33 # define _LIBCPP_VECTORIZE_ALGORITHMS 1 34 #else 35 # define _LIBCPP_VECTORIZE_ALGORITHMS 0 36 #endif 37 38 #if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 39 40 _LIBCPP_BEGIN_NAMESPACE_STD 41 42 template <class _Tp> 43 inline constexpr bool __can_map_to_integer_v = 44 sizeof(_Tp) == alignof(_Tp) && (sizeof(_Tp) == 1 || sizeof(_Tp) == 2 || sizeof(_Tp) == 4 || sizeof(_Tp) == 8); 45 46 template <size_t _TypeSize> 47 struct __get_as_integer_type_impl; 48 49 template <> 50 struct __get_as_integer_type_impl<1> { 51 using type = uint8_t; 52 }; 53 54 template <> 55 struct __get_as_integer_type_impl<2> { 56 using type = uint16_t; 57 }; 58 template <> 59 struct __get_as_integer_type_impl<4> { 60 using type = uint32_t; 61 }; 62 template <> 63 struct __get_as_integer_type_impl<8> { 64 using type = uint64_t; 65 }; 66 67 template <class _Tp> 68 using __get_as_integer_type_t = typename __get_as_integer_type_impl<sizeof(_Tp)>::type; 69 70 // This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance 71 // in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms 72 // as far as benchmarks are concerned. 73 # if defined(__AVX__) || defined(__MVS__) 74 template <class _Tp> 75 inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp); 76 # elif defined(__SSE__) || defined(__ARM_NEON__) 77 template <class _Tp> 78 inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp); 79 # elif defined(__MMX__) 80 template <class _Tp> 81 inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp); 82 # else 83 template <class _Tp> 84 inline constexpr size_t __native_vector_size = 1; 85 # endif 86 87 template <class _ArithmeticT, size_t _Np> 88 using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT; 89 90 template <class _VecT> 91 inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t { 92 static_assert(_False, "Not a vector!"); 93 }(); 94 95 template <class _Tp, size_t _Np> 96 inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np; 97 98 template <class _Tp, size_t _Np> 99 _LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) { 100 return _Tp{}; 101 } 102 103 template <class _VecT> 104 using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{})); 105 106 // This isn't inlined without always_inline when loading chars. 107 template <class _VecT, class _Iter> 108 _LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(_Iter __iter) noexcept { 109 return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept { 110 return _VecT{__iter[_Indices]...}; 111 }(make_index_sequence<__simd_vector_size_v<_VecT>>{}); 112 } 113 114 template <class _Tp, size_t _Np> 115 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept { 116 return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>)); 117 } 118 119 template <class _Tp, size_t _Np> 120 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept { 121 using __mask_vec = __simd_vector<bool, _Np>; 122 123 // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876 124 auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept { 125 # if defined(_LIBCPP_BIG_ENDIAN) 126 return std::min<size_t>( 127 _Np, std::__countl_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)))); 128 # else 129 return std::min<size_t>( 130 _Np, std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)))); 131 # endif 132 }; 133 134 if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) { 135 return __impl(uint8_t{}); 136 } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) { 137 return __impl(uint16_t{}); 138 } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) { 139 return __impl(uint32_t{}); 140 } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) { 141 return __impl(uint64_t{}); 142 } else { 143 static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type"); 144 return 0; 145 } 146 } 147 148 template <class _Tp, size_t _Np> 149 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept { 150 return std::__find_first_set(~__vec); 151 } 152 153 _LIBCPP_END_NAMESPACE_STD 154 155 #endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 156 157 _LIBCPP_POP_MACROS 158 159 #endif // _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H 160