1 /* 2 * Copyright (c) 2020 Netflix, Inc 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer, 9 * without modification. 10 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 11 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any 12 * redistribution must be conditioned upon including a substantially 13 * similar Disclaimer requirement for further binary redistribution. 14 * 15 * NO WARRANTY 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY 19 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 20 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, 21 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 24 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 * THE POSSIBILITY OF SUCH DAMAGES. 27 */ 28 29 /* 30 * A driver for the OpenCrypto framework which uses assembly routines 31 * from OpenSSL. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/types.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/module.h> 42 #include <machine/fpu.h> 43 #include <machine/md_var.h> 44 #include <x86/cputypes.h> 45 #include <x86/specialreg.h> 46 47 #include <opencrypto/cryptodev.h> 48 #include <opencrypto/xform_auth.h> 49 50 #include <crypto/openssl/ossl.h> 51 52 #include "cryptodev_if.h" 53 54 struct ossl_softc { 55 int32_t sc_cid; 56 }; 57 58 struct ossl_session_hash { 59 struct ossl_hash_context ictx; 60 struct ossl_hash_context octx; 61 struct auth_hash *axf; 62 u_int mlen; 63 }; 64 65 struct ossl_session { 66 struct ossl_session_hash hash; 67 }; 68 69 /* 70 * See OPENSSL_ia32cap(3). 71 * 72 * [0] = cpu_feature but with a few custom bits 73 * [1] = cpu_feature2 but with AMD XOP in bit 11 74 * [2] = cpu_stdext_feature 75 * [3] = 0 76 */ 77 unsigned int OPENSSL_ia32cap_P[4]; 78 79 static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto"); 80 81 static void 82 ossl_cpuid(void) 83 { 84 uint64_t xcr0; 85 u_int regs[4]; 86 u_int max_cores; 87 88 /* Derived from OpenSSL_ia32_cpuid. */ 89 90 OPENSSL_ia32cap_P[0] = cpu_feature & ~(CPUID_B20 | CPUID_IA64); 91 if (cpu_vendor_id == CPU_VENDOR_INTEL) { 92 OPENSSL_ia32cap_P[0] |= CPUID_IA64; 93 if ((cpu_id & 0xf00) != 0xf00) 94 OPENSSL_ia32cap_P[0] |= CPUID_B20; 95 } 96 97 /* Only leave CPUID_HTT on if HTT is present. */ 98 if (cpu_vendor_id == CPU_VENDOR_AMD && cpu_exthigh >= 0x80000008) { 99 max_cores = (cpu_procinfo2 & AMDID_CMP_CORES) + 1; 100 if (cpu_feature & CPUID_HTT) { 101 if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 <= max_cores) 102 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; 103 } 104 } else { 105 if (cpu_high >= 4) { 106 cpuid_count(4, 0, regs); 107 max_cores = (regs[0] >> 26) & 0xfff; 108 } else 109 max_cores = -1; 110 } 111 if (max_cores == 0) 112 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; 113 else if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 == 0) 114 OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; 115 116 OPENSSL_ia32cap_P[1] = cpu_feature2 & ~AMDID2_XOP; 117 if (cpu_vendor_id == CPU_VENDOR_AMD) 118 OPENSSL_ia32cap_P[1] |= amd_feature2 & AMDID2_XOP; 119 120 OPENSSL_ia32cap_P[2] = cpu_stdext_feature; 121 if ((OPENSSL_ia32cap_P[1] & CPUID2_XSAVE) == 0) 122 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F | 123 CPUID_STDEXT_AVX512DQ); 124 125 /* Disable AVX512F on Skylake-X. */ 126 if ((cpu_id & 0x0fff0ff0) == 0x00050650) 127 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F); 128 129 if (cpu_feature2 & CPUID2_OSXSAVE) 130 xcr0 = rxcr(0); 131 else 132 xcr0 = 0; 133 134 if ((xcr0 & (XFEATURE_AVX512 | XFEATURE_AVX)) != 135 (XFEATURE_AVX512 | XFEATURE_AVX)) 136 OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512VL | 137 CPUID_STDEXT_AVX512BW | CPUID_STDEXT_AVX512IFMA | 138 CPUID_STDEXT_AVX512F); 139 if ((xcr0 & XFEATURE_AVX) != XFEATURE_AVX) { 140 OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA); 141 OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2; 142 } 143 } 144 145 static void 146 ossl_identify(driver_t *driver, device_t parent) 147 { 148 149 if (device_find_child(parent, "ossl", -1) == NULL) 150 BUS_ADD_CHILD(parent, 10, "ossl", -1); 151 } 152 153 static int 154 ossl_probe(device_t dev) 155 { 156 157 device_set_desc(dev, "OpenSSL crypto"); 158 return (BUS_PROBE_DEFAULT); 159 } 160 161 static int 162 ossl_attach(device_t dev) 163 { 164 struct ossl_softc *sc; 165 166 sc = device_get_softc(dev); 167 168 ossl_cpuid(); 169 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session), 170 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | 171 CRYPTOCAP_F_ACCEL_SOFTWARE); 172 if (sc->sc_cid < 0) { 173 device_printf(dev, "failed to allocate crypto driver id\n"); 174 return (ENXIO); 175 } 176 177 return (0); 178 } 179 180 static int 181 ossl_detach(device_t dev) 182 { 183 struct ossl_softc *sc; 184 185 sc = device_get_softc(dev); 186 187 crypto_unregister_all(sc->sc_cid); 188 189 return (0); 190 } 191 192 static struct auth_hash * 193 ossl_lookup_hash(const struct crypto_session_params *csp) 194 { 195 196 switch (csp->csp_auth_alg) { 197 case CRYPTO_SHA1: 198 case CRYPTO_SHA1_HMAC: 199 return (&ossl_hash_sha1); 200 case CRYPTO_SHA2_224: 201 case CRYPTO_SHA2_224_HMAC: 202 return (&ossl_hash_sha224); 203 case CRYPTO_SHA2_256: 204 case CRYPTO_SHA2_256_HMAC: 205 return (&ossl_hash_sha256); 206 case CRYPTO_SHA2_384: 207 case CRYPTO_SHA2_384_HMAC: 208 return (&ossl_hash_sha384); 209 case CRYPTO_SHA2_512: 210 case CRYPTO_SHA2_512_HMAC: 211 return (&ossl_hash_sha512); 212 default: 213 return (NULL); 214 } 215 } 216 217 static int 218 ossl_probesession(device_t dev, const struct crypto_session_params *csp) 219 { 220 221 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 222 0) 223 return (EINVAL); 224 switch (csp->csp_mode) { 225 case CSP_MODE_DIGEST: 226 if (ossl_lookup_hash(csp) == NULL) 227 return (EINVAL); 228 break; 229 default: 230 return (EINVAL); 231 } 232 233 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 234 } 235 236 static void 237 ossl_setkey_hmac(struct ossl_session *s, const void *key, int klen) 238 { 239 240 hmac_init_ipad(s->hash.axf, key, klen, &s->hash.ictx); 241 hmac_init_opad(s->hash.axf, key, klen, &s->hash.octx); 242 } 243 244 static int 245 ossl_newsession(device_t dev, crypto_session_t cses, 246 const struct crypto_session_params *csp) 247 { 248 struct ossl_session *s; 249 struct auth_hash *axf; 250 251 s = crypto_get_driver_session(cses); 252 253 axf = ossl_lookup_hash(csp); 254 s->hash.axf = axf; 255 if (csp->csp_auth_mlen == 0) 256 s->hash.mlen = axf->hashsize; 257 else 258 s->hash.mlen = csp->csp_auth_mlen; 259 260 if (csp->csp_auth_klen == 0) { 261 axf->Init(&s->hash.ictx); 262 } else { 263 if (csp->csp_auth_key != NULL) { 264 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 265 ossl_setkey_hmac(s, csp->csp_auth_key, 266 csp->csp_auth_klen); 267 fpu_kern_leave(curthread, NULL); 268 } 269 } 270 return (0); 271 } 272 273 static int 274 ossl_process(device_t dev, struct cryptop *crp, int hint) 275 { 276 struct ossl_hash_context ctx; 277 char digest[HASH_MAX_LEN]; 278 const struct crypto_session_params *csp; 279 struct ossl_session *s; 280 struct auth_hash *axf; 281 int error; 282 bool fpu_entered; 283 284 s = crypto_get_driver_session(crp->crp_session); 285 csp = crypto_get_params(crp->crp_session); 286 axf = s->hash.axf; 287 288 if (is_fpu_kern_thread(0)) { 289 fpu_entered = false; 290 } else { 291 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 292 fpu_entered = true; 293 } 294 295 if (crp->crp_auth_key != NULL) 296 ossl_setkey_hmac(s, crp->crp_auth_key, csp->csp_auth_klen); 297 298 ctx = s->hash.ictx; 299 300 if (crp->crp_aad != NULL) 301 error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 302 else 303 error = crypto_apply(crp, crp->crp_aad_start, 304 crp->crp_aad_length, axf->Update, &ctx); 305 if (error) 306 goto out; 307 308 error = crypto_apply(crp, crp->crp_payload_start, 309 crp->crp_payload_length, axf->Update, &ctx); 310 if (error) 311 goto out; 312 313 axf->Final(digest, &ctx); 314 315 if (csp->csp_auth_klen != 0) { 316 ctx = s->hash.octx; 317 axf->Update(&ctx, digest, axf->hashsize); 318 axf->Final(digest, &ctx); 319 } 320 321 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 322 char digest2[HASH_MAX_LEN]; 323 324 crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen, 325 digest2); 326 if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0) 327 error = EBADMSG; 328 explicit_bzero(digest2, sizeof(digest2)); 329 } else { 330 crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen, 331 digest); 332 } 333 explicit_bzero(digest, sizeof(digest)); 334 335 out: 336 if (fpu_entered) 337 fpu_kern_leave(curthread, NULL); 338 339 crp->crp_etype = error; 340 crypto_done(crp); 341 342 explicit_bzero(&ctx, sizeof(ctx)); 343 return (0); 344 } 345 346 static device_method_t ossl_methods[] = { 347 DEVMETHOD(device_identify, ossl_identify), 348 DEVMETHOD(device_probe, ossl_probe), 349 DEVMETHOD(device_attach, ossl_attach), 350 DEVMETHOD(device_detach, ossl_detach), 351 352 DEVMETHOD(cryptodev_probesession, ossl_probesession), 353 DEVMETHOD(cryptodev_newsession, ossl_newsession), 354 DEVMETHOD(cryptodev_process, ossl_process), 355 356 DEVMETHOD_END 357 }; 358 359 static driver_t ossl_driver = { 360 "ossl", 361 ossl_methods, 362 sizeof(struct ossl_softc) 363 }; 364 365 static devclass_t ossl_devclass; 366 367 DRIVER_MODULE(ossl, nexus, ossl_driver, ossl_devclass, NULL, NULL); 368 MODULE_VERSION(ossl, 1); 369 MODULE_DEPEND(ossl, crypto, 1, 1, 1); 370