1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2020 Netflix, Inc
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer,
11 * without modification.
12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
13 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
14 * redistribution must be conditioned upon including a substantially
15 * similar Disclaimer requirement for further binary redistribution.
16 *
17 * NO WARRANTY
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
21 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
23 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
26 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
28 * THE POSSIBILITY OF SUCH DAMAGES.
29 */
30
31 #include <sys/types.h>
32 #include <sys/systm.h>
33
34 #include <machine/cpufunc.h>
35 #include <machine/md_var.h>
36 #include <x86/cputypes.h>
37 #include <x86/specialreg.h>
38
39 #include <crypto/openssl/ossl.h>
40 #include <crypto/openssl/ossl_aes_gcm.h>
41 #include <crypto/openssl/ossl_cipher.h>
42
43 /*
44 * See OPENSSL_ia32cap(3).
45 *
46 * [0] = cpu_feature but with a few custom bits
47 * [1] = cpu_feature2 but with AMD XOP in bit 11
48 * [2] = cpu_stdext_feature
49 * [3] = cpu_stdext_feature2
50 */
51 unsigned int OPENSSL_ia32cap_P[4];
52 #define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
53
54 ossl_cipher_setkey_t aesni_set_encrypt_key;
55 ossl_cipher_setkey_t aesni_set_decrypt_key;
56
57 #ifdef __amd64__
58 int ossl_vaes_vpclmulqdq_capable(void);
59 ossl_cipher_setkey_t ossl_aes_gcm_setkey_aesni;
60 ossl_cipher_setkey_t ossl_aes_gcm_setkey_avx512;
61 #endif
62
63 void
ossl_cpuid(struct ossl_softc * sc)64 ossl_cpuid(struct ossl_softc *sc)
65 {
66 uint64_t xcr0;
67 u_int regs[4];
68 u_int max_cores;
69
70 /* Derived from OpenSSL_ia32_cpuid. */
71
72 OPENSSL_ia32cap_P[0] = cpu_feature & ~(CPUID_B20 | CPUID_IA64);
73 if (cpu_vendor_id == CPU_VENDOR_INTEL) {
74 OPENSSL_ia32cap_P[0] |= CPUID_IA64;
75 if ((cpu_id & 0xf00) != 0xf00)
76 OPENSSL_ia32cap_P[0] |= CPUID_B20;
77 }
78
79 /* Only leave CPUID_HTT on if HTT is present. */
80 if (cpu_vendor_id == CPU_VENDOR_AMD && cpu_exthigh >= 0x80000008) {
81 max_cores = (cpu_procinfo2 & AMDID_CMP_CORES) + 1;
82 if (cpu_feature & CPUID_HTT) {
83 if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 <= max_cores)
84 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT;
85 }
86 } else {
87 if (cpu_high >= 4) {
88 cpuid_count(4, 0, regs);
89 max_cores = (regs[0] >> 26) & 0xfff;
90 } else
91 max_cores = -1;
92 }
93 if (max_cores == 0)
94 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT;
95 else if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 == 0)
96 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT;
97
98 OPENSSL_ia32cap_P[1] = cpu_feature2 & ~AMDID2_XOP;
99 if (cpu_vendor_id == CPU_VENDOR_AMD)
100 OPENSSL_ia32cap_P[1] |= amd_feature2 & AMDID2_XOP;
101
102 OPENSSL_ia32cap_P[2] = cpu_stdext_feature;
103 if ((OPENSSL_ia32cap_P[1] & CPUID2_XSAVE) == 0)
104 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F |
105 CPUID_STDEXT_AVX512DQ);
106
107 /* Disable AVX512F on Skylake-X. */
108 if ((cpu_id & 0x0fff0ff0) == 0x00050650)
109 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F);
110
111 if (cpu_feature2 & CPUID2_OSXSAVE)
112 xcr0 = rxcr(0);
113 else
114 xcr0 = 0;
115
116 if ((xcr0 & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
117 (XFEATURE_AVX512 | XFEATURE_AVX))
118 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512VL |
119 CPUID_STDEXT_AVX512BW | CPUID_STDEXT_AVX512IFMA |
120 CPUID_STDEXT_AVX512F);
121 if ((xcr0 & XFEATURE_AVX) != XFEATURE_AVX) {
122 OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA);
123 OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2;
124 }
125 OPENSSL_ia32cap_P[3] = cpu_stdext_feature2;
126
127 if (!AESNI_CAPABLE)
128 return;
129
130 sc->has_aes = true;
131 ossl_cipher_aes_cbc.set_encrypt_key = aesni_set_encrypt_key;
132 ossl_cipher_aes_cbc.set_decrypt_key = aesni_set_decrypt_key;
133
134 #ifdef __amd64__
135 if (ossl_vaes_vpclmulqdq_capable()) {
136 ossl_cipher_aes_gcm.set_encrypt_key =
137 ossl_aes_gcm_setkey_avx512;
138 ossl_cipher_aes_gcm.set_decrypt_key =
139 ossl_aes_gcm_setkey_avx512;
140 sc->has_aes_gcm = true;
141 } else if ((cpu_feature2 &
142 (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) ==
143 (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) {
144 ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey_aesni;
145 ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey_aesni;
146 sc->has_aes_gcm = true;
147 } else {
148 sc->has_aes_gcm = false;
149 }
150 #else
151 sc->has_aes_gcm = false;
152 #endif
153 }
154