1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020 Netflix, Inc 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any 14 * redistribution must be conditioned upon including a substantially 15 * similar Disclaimer requirement for further binary redistribution. 16 * 17 * NO WARRANTY 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY 21 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, 23 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 26 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 * THE POSSIBILITY OF SUCH DAMAGES. 29 */ 30 31 /* 32 * A driver for the OpenCrypto framework which uses assembly routines 33 * from OpenSSL. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/types.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/malloc.h> 43 #include <sys/module.h> 44 45 #include <machine/fpu.h> 46 47 #include <opencrypto/cryptodev.h> 48 #include <opencrypto/xform_auth.h> 49 50 #include <crypto/openssl/ossl.h> 51 #include <crypto/openssl/ossl_chacha.h> 52 #include <crypto/openssl/ossl_cipher.h> 53 54 #include "cryptodev_if.h" 55 56 static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto"); 57 58 static void 59 ossl_identify(driver_t *driver, device_t parent) 60 { 61 62 if (device_find_child(parent, "ossl", -1) == NULL) 63 BUS_ADD_CHILD(parent, 10, "ossl", -1); 64 } 65 66 static int 67 ossl_probe(device_t dev) 68 { 69 70 device_set_desc(dev, "OpenSSL crypto"); 71 return (BUS_PROBE_DEFAULT); 72 } 73 74 static int 75 ossl_attach(device_t dev) 76 { 77 struct ossl_softc *sc; 78 79 sc = device_get_softc(dev); 80 81 sc->has_aes = sc->has_aes_gcm = false; 82 83 ossl_cpuid(sc); 84 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session), 85 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | 86 CRYPTOCAP_F_ACCEL_SOFTWARE); 87 if (sc->sc_cid < 0) { 88 device_printf(dev, "failed to allocate crypto driver id\n"); 89 return (ENXIO); 90 } 91 92 return (0); 93 } 94 95 static int 96 ossl_detach(device_t dev) 97 { 98 struct ossl_softc *sc; 99 100 sc = device_get_softc(dev); 101 102 crypto_unregister_all(sc->sc_cid); 103 104 return (0); 105 } 106 107 static struct auth_hash * 108 ossl_lookup_hash(const struct crypto_session_params *csp) 109 { 110 111 switch (csp->csp_auth_alg) { 112 case CRYPTO_SHA1: 113 case CRYPTO_SHA1_HMAC: 114 return (&ossl_hash_sha1); 115 case CRYPTO_SHA2_224: 116 case CRYPTO_SHA2_224_HMAC: 117 return (&ossl_hash_sha224); 118 case CRYPTO_SHA2_256: 119 case CRYPTO_SHA2_256_HMAC: 120 return (&ossl_hash_sha256); 121 case CRYPTO_SHA2_384: 122 case CRYPTO_SHA2_384_HMAC: 123 return (&ossl_hash_sha384); 124 case CRYPTO_SHA2_512: 125 case CRYPTO_SHA2_512_HMAC: 126 return (&ossl_hash_sha512); 127 case CRYPTO_POLY1305: 128 return (&ossl_hash_poly1305); 129 default: 130 return (NULL); 131 } 132 } 133 134 static struct ossl_cipher* 135 ossl_lookup_cipher(const struct crypto_session_params *csp) 136 { 137 138 switch (csp->csp_cipher_alg) { 139 case CRYPTO_AES_CBC: 140 switch (csp->csp_cipher_klen * 8) { 141 case 128: 142 case 192: 143 case 256: 144 break; 145 default: 146 return (NULL); 147 } 148 return (&ossl_cipher_aes_cbc); 149 case CRYPTO_AES_NIST_GCM_16: 150 switch (csp->csp_cipher_klen * 8) { 151 case 128: 152 case 192: 153 case 256: 154 break; 155 default: 156 return (NULL); 157 } 158 return (&ossl_cipher_aes_gcm); 159 case CRYPTO_CHACHA20: 160 if (csp->csp_cipher_klen != CHACHA_KEY_SIZE) 161 return (NULL); 162 return (&ossl_cipher_chacha20); 163 default: 164 return (NULL); 165 } 166 } 167 168 static int 169 ossl_probesession(device_t dev, const struct crypto_session_params *csp) 170 { 171 struct ossl_softc *sc = device_get_softc(dev); 172 173 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 174 0) 175 return (EINVAL); 176 switch (csp->csp_mode) { 177 case CSP_MODE_DIGEST: 178 if (ossl_lookup_hash(csp) == NULL) 179 return (EINVAL); 180 break; 181 case CSP_MODE_CIPHER: 182 if (csp->csp_cipher_alg != CRYPTO_CHACHA20 && !sc->has_aes) 183 return (EINVAL); 184 if (ossl_lookup_cipher(csp) == NULL) 185 return (EINVAL); 186 break; 187 case CSP_MODE_ETA: 188 if (!sc->has_aes || 189 csp->csp_cipher_alg == CRYPTO_CHACHA20 || 190 ossl_lookup_hash(csp) == NULL || 191 ossl_lookup_cipher(csp) == NULL) 192 return (EINVAL); 193 break; 194 case CSP_MODE_AEAD: 195 switch (csp->csp_cipher_alg) { 196 case CRYPTO_CHACHA20_POLY1305: 197 break; 198 case CRYPTO_AES_NIST_GCM_16: 199 if (!sc->has_aes_gcm || ossl_lookup_cipher(csp) == NULL) 200 return (EINVAL); 201 if (csp->csp_ivlen != AES_GCM_IV_LEN) 202 return (EINVAL); 203 if (csp->csp_auth_mlen != 0 && 204 csp->csp_auth_mlen != GMAC_DIGEST_LEN) 205 return (EINVAL); 206 break; 207 default: 208 return (EINVAL); 209 } 210 break; 211 default: 212 return (EINVAL); 213 } 214 215 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 216 } 217 218 static void 219 ossl_newsession_hash(struct ossl_session *s, 220 const struct crypto_session_params *csp) 221 { 222 struct auth_hash *axf; 223 224 axf = ossl_lookup_hash(csp); 225 s->hash.axf = axf; 226 if (csp->csp_auth_mlen == 0) 227 s->hash.mlen = axf->hashsize; 228 else 229 s->hash.mlen = csp->csp_auth_mlen; 230 231 if (csp->csp_auth_klen == 0) { 232 axf->Init(&s->hash.ictx); 233 } else { 234 if (csp->csp_auth_key != NULL) { 235 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 236 if (axf->Setkey != NULL) { 237 axf->Init(&s->hash.ictx); 238 axf->Setkey(&s->hash.ictx, csp->csp_auth_key, 239 csp->csp_auth_klen); 240 } else { 241 hmac_init_ipad(axf, csp->csp_auth_key, 242 csp->csp_auth_klen, &s->hash.ictx); 243 hmac_init_opad(axf, csp->csp_auth_key, 244 csp->csp_auth_klen, &s->hash.octx); 245 } 246 fpu_kern_leave(curthread, NULL); 247 } 248 } 249 } 250 251 static int 252 ossl_newsession_cipher(struct ossl_session *s, 253 const struct crypto_session_params *csp) 254 { 255 struct ossl_cipher *cipher; 256 int error = 0; 257 258 cipher = ossl_lookup_cipher(csp); 259 if (cipher == NULL) 260 return (EINVAL); 261 262 s->cipher.cipher = cipher; 263 264 if (csp->csp_cipher_key == NULL) 265 return (0); 266 267 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 268 if (cipher->set_encrypt_key != NULL) { 269 error = cipher->set_encrypt_key(csp->csp_cipher_key, 270 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx); 271 if (error != 0) { 272 fpu_kern_leave(curthread, NULL); 273 return (error); 274 } 275 } 276 if (cipher->set_decrypt_key != NULL) 277 error = cipher->set_decrypt_key(csp->csp_cipher_key, 278 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx); 279 fpu_kern_leave(curthread, NULL); 280 281 return (error); 282 } 283 284 static int 285 ossl_newsession(device_t dev, crypto_session_t cses, 286 const struct crypto_session_params *csp) 287 { 288 struct ossl_session *s; 289 int error = 0; 290 291 s = crypto_get_driver_session(cses); 292 switch (csp->csp_mode) { 293 case CSP_MODE_DIGEST: 294 ossl_newsession_hash(s, csp); 295 break; 296 case CSP_MODE_CIPHER: 297 error = ossl_newsession_cipher(s, csp); 298 break; 299 case CSP_MODE_ETA: 300 ossl_newsession_hash(s, csp); 301 error = ossl_newsession_cipher(s, csp); 302 break; 303 case CSP_MODE_AEAD: 304 if (csp->csp_cipher_alg != CRYPTO_CHACHA20_POLY1305) 305 error = ossl_newsession_cipher(s, csp); 306 break; 307 default: 308 __assert_unreachable(); 309 } 310 311 return (error); 312 } 313 314 static int 315 ossl_process_hash(struct ossl_session *s, struct cryptop *crp, 316 const struct crypto_session_params *csp) 317 { 318 struct ossl_hash_context ctx; 319 char digest[HASH_MAX_LEN]; 320 struct auth_hash *axf; 321 int error; 322 323 axf = s->hash.axf; 324 325 if (crp->crp_auth_key == NULL) { 326 ctx = s->hash.ictx; 327 } else { 328 if (axf->Setkey != NULL) { 329 axf->Init(&ctx); 330 axf->Setkey(&ctx, crp->crp_auth_key, 331 csp->csp_auth_klen); 332 } else { 333 hmac_init_ipad(axf, crp->crp_auth_key, 334 csp->csp_auth_klen, &ctx); 335 } 336 } 337 338 if (crp->crp_aad != NULL) 339 error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 340 else 341 error = crypto_apply(crp, crp->crp_aad_start, 342 crp->crp_aad_length, axf->Update, &ctx); 343 if (error) 344 goto out; 345 346 error = crypto_apply(crp, crp->crp_payload_start, 347 crp->crp_payload_length, axf->Update, &ctx); 348 if (error) 349 goto out; 350 351 axf->Final(digest, &ctx); 352 353 if (csp->csp_auth_klen != 0 && axf->Setkey == NULL) { 354 if (crp->crp_auth_key == NULL) 355 ctx = s->hash.octx; 356 else 357 hmac_init_opad(axf, crp->crp_auth_key, 358 csp->csp_auth_klen, &ctx); 359 axf->Update(&ctx, digest, axf->hashsize); 360 axf->Final(digest, &ctx); 361 } 362 363 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 364 char digest2[HASH_MAX_LEN]; 365 366 crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen, 367 digest2); 368 if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0) 369 error = EBADMSG; 370 explicit_bzero(digest2, sizeof(digest2)); 371 } else { 372 crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen, 373 digest); 374 } 375 explicit_bzero(digest, sizeof(digest)); 376 377 out: 378 explicit_bzero(&ctx, sizeof(ctx)); 379 return (error); 380 } 381 382 static int 383 ossl_process_cipher(struct ossl_session *s, struct cryptop *crp, 384 const struct crypto_session_params *csp) 385 { 386 return (s->cipher.cipher->process(&s->cipher, crp, csp)); 387 } 388 389 static int 390 ossl_process_eta(struct ossl_session *s, struct cryptop *crp, 391 const struct crypto_session_params *csp) 392 { 393 int error; 394 395 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 396 error = s->cipher.cipher->process(&s->cipher, crp, csp); 397 if (error == 0) 398 error = ossl_process_hash(s, crp, csp); 399 } else { 400 error = ossl_process_hash(s, crp, csp); 401 if (error == 0) 402 error = s->cipher.cipher->process(&s->cipher, crp, csp); 403 } 404 405 return (error); 406 } 407 408 static int 409 ossl_process_aead(struct ossl_session *s, struct cryptop *crp, 410 const struct crypto_session_params *csp) 411 { 412 if (csp->csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) { 413 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 414 return (ossl_chacha20_poly1305_encrypt(crp, csp)); 415 else 416 return (ossl_chacha20_poly1305_decrypt(crp, csp)); 417 } else { 418 return (s->cipher.cipher->process(&s->cipher, crp, csp)); 419 } 420 } 421 422 static int 423 ossl_process(device_t dev, struct cryptop *crp, int hint) 424 { 425 const struct crypto_session_params *csp; 426 struct ossl_session *s; 427 int error; 428 bool fpu_entered; 429 430 s = crypto_get_driver_session(crp->crp_session); 431 csp = crypto_get_params(crp->crp_session); 432 433 if (is_fpu_kern_thread(0)) { 434 fpu_entered = false; 435 } else { 436 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 437 fpu_entered = true; 438 } 439 440 switch (csp->csp_mode) { 441 case CSP_MODE_DIGEST: 442 error = ossl_process_hash(s, crp, csp); 443 break; 444 case CSP_MODE_CIPHER: 445 error = ossl_process_cipher(s, crp, csp); 446 break; 447 case CSP_MODE_ETA: 448 error = ossl_process_eta(s, crp, csp); 449 break; 450 case CSP_MODE_AEAD: 451 error = ossl_process_aead(s, crp, csp); 452 break; 453 default: 454 __assert_unreachable(); 455 } 456 457 if (fpu_entered) 458 fpu_kern_leave(curthread, NULL); 459 460 crp->crp_etype = error; 461 crypto_done(crp); 462 463 return (0); 464 } 465 466 static device_method_t ossl_methods[] = { 467 DEVMETHOD(device_identify, ossl_identify), 468 DEVMETHOD(device_probe, ossl_probe), 469 DEVMETHOD(device_attach, ossl_attach), 470 DEVMETHOD(device_detach, ossl_detach), 471 472 DEVMETHOD(cryptodev_probesession, ossl_probesession), 473 DEVMETHOD(cryptodev_newsession, ossl_newsession), 474 DEVMETHOD(cryptodev_process, ossl_process), 475 476 DEVMETHOD_END 477 }; 478 479 static driver_t ossl_driver = { 480 "ossl", 481 ossl_methods, 482 sizeof(struct ossl_softc) 483 }; 484 485 DRIVER_MODULE(ossl, nexus, ossl_driver, NULL, NULL); 486 MODULE_VERSION(ossl, 1); 487 MODULE_DEPEND(ossl, crypto, 1, 1, 1); 488