1 #include <stddef.h>
2 #include <stdint.h>
3 #ifdef HAVE_ANDROID_GETCPUFEATURES
4 # include <cpu-features.h>
5 #endif
6
7 #include "private/common.h"
8 #include "runtime.h"
9
10 typedef struct CPUFeatures_ {
11 int initialized;
12 int has_neon;
13 int has_sse2;
14 int has_sse3;
15 int has_ssse3;
16 int has_sse41;
17 int has_avx;
18 int has_avx2;
19 int has_avx512f;
20 int has_pclmul;
21 int has_aesni;
22 int has_rdrand;
23 } CPUFeatures;
24
25 static CPUFeatures _cpu_features;
26
27 #define CPUID_EBX_AVX2 0x00000020
28 #define CPUID_EBX_AVX512F 0x00010000
29
30 #define CPUID_ECX_SSE3 0x00000001
31 #define CPUID_ECX_PCLMUL 0x00000002
32 #define CPUID_ECX_SSSE3 0x00000200
33 #define CPUID_ECX_SSE41 0x00080000
34 #define CPUID_ECX_AESNI 0x02000000
35 #define CPUID_ECX_XSAVE 0x04000000
36 #define CPUID_ECX_OSXSAVE 0x08000000
37 #define CPUID_ECX_AVX 0x10000000
38 #define CPUID_ECX_RDRAND 0x40000000
39
40 #define CPUID_EDX_SSE2 0x04000000
41
42 #define XCR0_SSE 0x00000002
43 #define XCR0_AVX 0x00000004
44
45 static int
_sodium_runtime_arm_cpu_features(CPUFeatures * const cpu_features)46 _sodium_runtime_arm_cpu_features(CPUFeatures * const cpu_features)
47 {
48 #ifndef __arm__
49 cpu_features->has_neon = 0;
50 return -1;
51 #else
52 # ifdef __APPLE__
53 # ifdef __ARM_NEON__
54 cpu_features->has_neon = 1;
55 # else
56 cpu_features->has_neon = 0;
57 # endif
58 # elif defined(HAVE_ANDROID_GETCPUFEATURES) && \
59 defined(ANDROID_CPU_ARM_FEATURE_NEON)
60 cpu_features->has_neon =
61 (android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0x0;
62 # else
63 cpu_features->has_neon = 0;
64 # endif
65 return 0;
66 #endif
67 }
68
69 static void
_cpuid(unsigned int cpu_info[4U],const unsigned int cpu_info_type)70 _cpuid(unsigned int cpu_info[4U], const unsigned int cpu_info_type)
71 {
72 #if defined(_MSC_VER) && \
73 (defined(_M_X64) || defined(_M_AMD64) || defined(_M_IX86))
74 __cpuid((int *) cpu_info, cpu_info_type);
75 #elif defined(HAVE_CPUID)
76 cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0;
77 # ifdef __i386__
78 __asm__ __volatile__(
79 "pushfl; pushfl; "
80 "popl %0; "
81 "movl %0, %1; xorl %2, %0; "
82 "pushl %0; "
83 "popfl; pushfl; popl %0; popfl"
84 : "=&r"(cpu_info[0]), "=&r"(cpu_info[1])
85 : "i"(0x200000));
86 if (((cpu_info[0] ^ cpu_info[1]) & 0x200000) == 0x0) {
87 return; /* LCOV_EXCL_LINE */
88 }
89 # endif
90 # ifdef __i386__
91 __asm__ __volatile__("xchgl %%ebx, %k1; cpuid; xchgl %%ebx, %k1"
92 : "=a"(cpu_info[0]), "=&r"(cpu_info[1]),
93 "=c"(cpu_info[2]), "=d"(cpu_info[3])
94 : "0"(cpu_info_type), "2"(0U));
95 # elif defined(__x86_64__)
96 __asm__ __volatile__("xchgq %%rbx, %q1; cpuid; xchgq %%rbx, %q1"
97 : "=a"(cpu_info[0]), "=&r"(cpu_info[1]),
98 "=c"(cpu_info[2]), "=d"(cpu_info[3])
99 : "0"(cpu_info_type), "2"(0U));
100 # else
101 __asm__ __volatile__("cpuid"
102 : "=a"(cpu_info[0]), "=b"(cpu_info[1]),
103 "=c"(cpu_info[2]), "=d"(cpu_info[3])
104 : "0"(cpu_info_type), "2"(0U));
105 # endif
106 #else
107 (void) cpu_info_type;
108 cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0;
109 #endif
110 }
111
112 static int
_sodium_runtime_intel_cpu_features(CPUFeatures * const cpu_features)113 _sodium_runtime_intel_cpu_features(CPUFeatures * const cpu_features)
114 {
115 unsigned int cpu_info[4];
116 unsigned int id;
117
118 _cpuid(cpu_info, 0x0);
119 if ((id = cpu_info[0]) == 0U) {
120 return -1; /* LCOV_EXCL_LINE */
121 }
122 _cpuid(cpu_info, 0x00000001);
123 #ifdef HAVE_EMMINTRIN_H
124 cpu_features->has_sse2 = ((cpu_info[3] & CPUID_EDX_SSE2) != 0x0);
125 #else
126 cpu_features->has_sse2 = 0;
127 #endif
128
129 #ifdef HAVE_PMMINTRIN_H
130 cpu_features->has_sse3 = ((cpu_info[2] & CPUID_ECX_SSE3) != 0x0);
131 #else
132 cpu_features->has_sse3 = 0;
133 #endif
134
135 #ifdef HAVE_TMMINTRIN_H
136 cpu_features->has_ssse3 = ((cpu_info[2] & CPUID_ECX_SSSE3) != 0x0);
137 #else
138 cpu_features->has_ssse3 = 0;
139 #endif
140
141 #ifdef HAVE_SMMINTRIN_H
142 cpu_features->has_sse41 = ((cpu_info[2] & CPUID_ECX_SSE41) != 0x0);
143 #else
144 cpu_features->has_sse41 = 0;
145 #endif
146
147 cpu_features->has_avx = 0;
148 #ifdef HAVE_AVXINTRIN_H
149 if ((cpu_info[2] & (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) ==
150 (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) {
151 uint32_t xcr0 = 0U;
152 # if defined(HAVE__XGETBV) || \
153 (defined(_MSC_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) && _MSC_FULL_VER >= 160040219)
154 xcr0 = (uint32_t) _xgetbv(0);
155 # elif defined(_MSC_VER) && defined(_M_IX86)
156 /*
157 * Visual Studio documentation states that eax/ecx/edx don't need to
158 * be preserved in inline assembly code. But that doesn't seem to
159 * always hold true on Visual Studio 2010.
160 */
161 __asm {
162 push eax
163 push ecx
164 push edx
165 xor ecx, ecx
166 _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0
167 mov xcr0, eax
168 pop edx
169 pop ecx
170 pop eax
171 }
172 # elif defined(HAVE_AVX_ASM)
173 __asm__ __volatile__(".byte 0x0f, 0x01, 0xd0" /* XGETBV */
174 : "=a"(xcr0)
175 : "c"((uint32_t) 0U)
176 : "%edx");
177 # endif
178 if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) {
179 cpu_features->has_avx = 1;
180 }
181 }
182 #endif
183
184 cpu_features->has_avx2 = 0;
185 #ifdef HAVE_AVX2INTRIN_H
186 if (cpu_features->has_avx) {
187 unsigned int cpu_info7[4];
188
189 _cpuid(cpu_info7, 0x00000007);
190 cpu_features->has_avx2 = ((cpu_info7[1] & CPUID_EBX_AVX2) != 0x0);
191 }
192 #endif
193
194 cpu_features->has_avx512f = 0;
195 #ifdef HAVE_AVX512FINTRIN_H
196 if (cpu_features->has_avx2) {
197 unsigned int cpu_info7[4];
198
199 _cpuid(cpu_info7, 0x00000007);
200 cpu_features->has_avx512f = ((cpu_info7[1] & CPUID_EBX_AVX512F) != 0x0);
201 }
202 #endif
203
204 #ifdef HAVE_WMMINTRIN_H
205 cpu_features->has_pclmul = ((cpu_info[2] & CPUID_ECX_PCLMUL) != 0x0);
206 cpu_features->has_aesni = ((cpu_info[2] & CPUID_ECX_AESNI) != 0x0);
207 #else
208 cpu_features->has_pclmul = 0;
209 cpu_features->has_aesni = 0;
210 #endif
211
212 #ifdef HAVE_RDRAND
213 cpu_features->has_rdrand = ((cpu_info[2] & CPUID_ECX_RDRAND) != 0x0);
214 #else
215 cpu_features->has_rdrand = 0;
216 #endif
217
218 return 0;
219 }
220
221 int
_sodium_runtime_get_cpu_features(void)222 _sodium_runtime_get_cpu_features(void)
223 {
224 int ret = -1;
225
226 ret &= _sodium_runtime_arm_cpu_features(&_cpu_features);
227 ret &= _sodium_runtime_intel_cpu_features(&_cpu_features);
228 _cpu_features.initialized = 1;
229
230 return ret;
231 }
232
233 int
sodium_runtime_has_neon(void)234 sodium_runtime_has_neon(void)
235 {
236 return _cpu_features.has_neon;
237 }
238
239 int
sodium_runtime_has_sse2(void)240 sodium_runtime_has_sse2(void)
241 {
242 return _cpu_features.has_sse2;
243 }
244
245 int
sodium_runtime_has_sse3(void)246 sodium_runtime_has_sse3(void)
247 {
248 return _cpu_features.has_sse3;
249 }
250
251 int
sodium_runtime_has_ssse3(void)252 sodium_runtime_has_ssse3(void)
253 {
254 return _cpu_features.has_ssse3;
255 }
256
257 int
sodium_runtime_has_sse41(void)258 sodium_runtime_has_sse41(void)
259 {
260 return _cpu_features.has_sse41;
261 }
262
263 int
sodium_runtime_has_avx(void)264 sodium_runtime_has_avx(void)
265 {
266 return _cpu_features.has_avx;
267 }
268
269 int
sodium_runtime_has_avx2(void)270 sodium_runtime_has_avx2(void)
271 {
272 return _cpu_features.has_avx2;
273 }
274
275 int
sodium_runtime_has_avx512f(void)276 sodium_runtime_has_avx512f(void)
277 {
278 return _cpu_features.has_avx512f;
279 }
280
281 int
sodium_runtime_has_pclmul(void)282 sodium_runtime_has_pclmul(void)
283 {
284 return _cpu_features.has_pclmul;
285 }
286
287 int
sodium_runtime_has_aesni(void)288 sodium_runtime_has_aesni(void)
289 {
290 return _cpu_features.has_aesni;
291 }
292
293 int
sodium_runtime_has_rdrand(void)294 sodium_runtime_has_rdrand(void)
295 {
296 return _cpu_features.has_rdrand;
297 }
298