1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020 Netflix, Inc 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any 14 * redistribution must be conditioned upon including a substantially 15 * similar Disclaimer requirement for further binary redistribution. 16 * 17 * NO WARRANTY 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY 21 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, 23 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 26 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 * THE POSSIBILITY OF SUCH DAMAGES. 29 * 30 * $FreeBSD$ 31 */ 32 33 #include <sys/types.h> 34 #include <sys/systm.h> 35 36 #include <machine/cpufunc.h> 37 #include <machine/md_var.h> 38 #include <x86/cputypes.h> 39 #include <x86/specialreg.h> 40 41 #include <crypto/openssl/ossl.h> 42 #include <crypto/openssl/ossl_aes_gcm.h> 43 #include <crypto/openssl/ossl_cipher.h> 44 45 /* 46 * See OPENSSL_ia32cap(3). 47 * 48 * [0] = cpu_feature but with a few custom bits 49 * [1] = cpu_feature2 but with AMD XOP in bit 11 50 * [2] = cpu_stdext_feature 51 * [3] = cpu_stdext_feature2 52 */ 53 unsigned int OPENSSL_ia32cap_P[4]; 54 #define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32))) 55 56 ossl_cipher_setkey_t aesni_set_encrypt_key; 57 ossl_cipher_setkey_t aesni_set_decrypt_key; 58 59 #ifdef __amd64__ 60 int ossl_vaes_vpclmulqdq_capable(void); 61 ossl_cipher_setkey_t ossl_aes_gcm_setkey_aesni; 62 ossl_cipher_setkey_t ossl_aes_gcm_setkey_avx512; 63 #endif 64 65 void 66 ossl_cpuid(struct ossl_softc *sc) 67 { 68 uint64_t xcr0; 69 u_int regs[4]; 70 u_int max_cores; 71 72 /* Derived from OpenSSL_ia32_cpuid. */ 73 74 OPENSSL_ia32cap_P[0] = cpu_feature & ~(CPUID_B20 | CPUID_IA64); 75 if (cpu_vendor_id == CPU_VENDOR_INTEL) { 76 OPENSSL_ia32cap_P[0] |= CPUID_IA64; 77 if ((cpu_id & 0xf00) != 0xf00) 78 OPENSSL_ia32cap_P[0] |= CPUID_B20; 79 } 80 81 /* Only leave CPUID_HTT on if HTT is present. */ 82 if (cpu_vendor_id == CPU_VENDOR_AMD && cpu_exthigh >= 0x80000008) { 83 max_cores = (cpu_procinfo2 & AMDID_CMP_CORES) + 1; 84 if (cpu_feature & CPUID_HTT) { 85 if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 <= max_cores) 86 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; 87 } 88 } else { 89 if (cpu_high >= 4) { 90 cpuid_count(4, 0, regs); 91 max_cores = (regs[0] >> 26) & 0xfff; 92 } else 93 max_cores = -1; 94 } 95 if (max_cores == 0) 96 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; 97 else if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 == 0) 98 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; 99 100 OPENSSL_ia32cap_P[1] = cpu_feature2 & ~AMDID2_XOP; 101 if (cpu_vendor_id == CPU_VENDOR_AMD) 102 OPENSSL_ia32cap_P[1] |= amd_feature2 & AMDID2_XOP; 103 104 OPENSSL_ia32cap_P[2] = cpu_stdext_feature; 105 if ((OPENSSL_ia32cap_P[1] & CPUID2_XSAVE) == 0) 106 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F | 107 CPUID_STDEXT_AVX512DQ); 108 109 /* Disable AVX512F on Skylake-X. */ 110 if ((cpu_id & 0x0fff0ff0) == 0x00050650) 111 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F); 112 113 if (cpu_feature2 & CPUID2_OSXSAVE) 114 xcr0 = rxcr(0); 115 else 116 xcr0 = 0; 117 118 if ((xcr0 & (XFEATURE_AVX512 | XFEATURE_AVX)) != 119 (XFEATURE_AVX512 | XFEATURE_AVX)) 120 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512VL | 121 CPUID_STDEXT_AVX512BW | CPUID_STDEXT_AVX512IFMA | 122 CPUID_STDEXT_AVX512F); 123 if ((xcr0 & XFEATURE_AVX) != XFEATURE_AVX) { 124 OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA); 125 OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2; 126 } 127 OPENSSL_ia32cap_P[3] = cpu_stdext_feature2; 128 129 if (!AESNI_CAPABLE) 130 return; 131 132 sc->has_aes = true; 133 ossl_cipher_aes_cbc.set_encrypt_key = aesni_set_encrypt_key; 134 ossl_cipher_aes_cbc.set_decrypt_key = aesni_set_decrypt_key; 135 136 #ifdef __amd64__ 137 if (ossl_vaes_vpclmulqdq_capable()) { 138 ossl_cipher_aes_gcm.set_encrypt_key = 139 ossl_aes_gcm_setkey_avx512; 140 ossl_cipher_aes_gcm.set_decrypt_key = 141 ossl_aes_gcm_setkey_avx512; 142 sc->has_aes_gcm = true; 143 } else if ((cpu_feature2 & 144 (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) == 145 (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) { 146 ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey_aesni; 147 ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey_aesni; 148 sc->has_aes_gcm = true; 149 } else { 150 sc->has_aes_gcm = false; 151 } 152 #else 153 sc->has_aes_gcm = false; 154 #endif 155 } 156