1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020 Netflix, Inc 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any 14 * redistribution must be conditioned upon including a substantially 15 * similar Disclaimer requirement for further binary redistribution. 16 * 17 * NO WARRANTY 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY 21 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, 23 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 26 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 * THE POSSIBILITY OF SUCH DAMAGES. 29 */ 30 31 /* 32 * A driver for the OpenCrypto framework which uses assembly routines 33 * from OpenSSL. 34 */ 35 36 #include <sys/cdefs.h> 37 #include <sys/types.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/module.h> 42 43 #include <machine/fpu.h> 44 45 #include <opencrypto/cryptodev.h> 46 #include <opencrypto/xform_auth.h> 47 48 #include <crypto/openssl/ossl.h> 49 #include <crypto/openssl/ossl_chacha.h> 50 #include <crypto/openssl/ossl_cipher.h> 51 52 #include "cryptodev_if.h" 53 54 static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto"); 55 56 static void 57 ossl_identify(driver_t *driver, device_t parent) 58 { 59 60 if (device_find_child(parent, "ossl", -1) == NULL) 61 BUS_ADD_CHILD(parent, 10, "ossl", -1); 62 } 63 64 static int 65 ossl_probe(device_t dev) 66 { 67 68 device_set_desc(dev, "OpenSSL crypto"); 69 return (BUS_PROBE_DEFAULT); 70 } 71 72 static int 73 ossl_attach(device_t dev) 74 { 75 struct ossl_softc *sc; 76 77 sc = device_get_softc(dev); 78 79 sc->has_aes = sc->has_aes_gcm = false; 80 81 ossl_cpuid(sc); 82 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session), 83 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | 84 CRYPTOCAP_F_ACCEL_SOFTWARE); 85 if (sc->sc_cid < 0) { 86 device_printf(dev, "failed to allocate crypto driver id\n"); 87 return (ENXIO); 88 } 89 90 return (0); 91 } 92 93 static int 94 ossl_detach(device_t dev) 95 { 96 struct ossl_softc *sc; 97 98 sc = device_get_softc(dev); 99 100 crypto_unregister_all(sc->sc_cid); 101 102 return (0); 103 } 104 105 static struct auth_hash * 106 ossl_lookup_hash(const struct crypto_session_params *csp) 107 { 108 109 switch (csp->csp_auth_alg) { 110 case CRYPTO_SHA1: 111 case CRYPTO_SHA1_HMAC: 112 return (&ossl_hash_sha1); 113 case CRYPTO_SHA2_224: 114 case CRYPTO_SHA2_224_HMAC: 115 return (&ossl_hash_sha224); 116 case CRYPTO_SHA2_256: 117 case CRYPTO_SHA2_256_HMAC: 118 return (&ossl_hash_sha256); 119 case CRYPTO_SHA2_384: 120 case CRYPTO_SHA2_384_HMAC: 121 return (&ossl_hash_sha384); 122 case CRYPTO_SHA2_512: 123 case CRYPTO_SHA2_512_HMAC: 124 return (&ossl_hash_sha512); 125 case CRYPTO_POLY1305: 126 return (&ossl_hash_poly1305); 127 default: 128 return (NULL); 129 } 130 } 131 132 static struct ossl_cipher* 133 ossl_lookup_cipher(const struct crypto_session_params *csp) 134 { 135 136 switch (csp->csp_cipher_alg) { 137 case CRYPTO_AES_CBC: 138 switch (csp->csp_cipher_klen * 8) { 139 case 128: 140 case 192: 141 case 256: 142 break; 143 default: 144 return (NULL); 145 } 146 return (&ossl_cipher_aes_cbc); 147 case CRYPTO_AES_NIST_GCM_16: 148 switch (csp->csp_cipher_klen * 8) { 149 case 128: 150 case 192: 151 case 256: 152 break; 153 default: 154 return (NULL); 155 } 156 return (&ossl_cipher_aes_gcm); 157 case CRYPTO_CHACHA20: 158 if (csp->csp_cipher_klen != CHACHA_KEY_SIZE) 159 return (NULL); 160 return (&ossl_cipher_chacha20); 161 default: 162 return (NULL); 163 } 164 } 165 166 static int 167 ossl_probesession(device_t dev, const struct crypto_session_params *csp) 168 { 169 struct ossl_softc *sc = device_get_softc(dev); 170 171 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 172 0) 173 return (EINVAL); 174 switch (csp->csp_mode) { 175 case CSP_MODE_DIGEST: 176 if (ossl_lookup_hash(csp) == NULL) 177 return (EINVAL); 178 break; 179 case CSP_MODE_CIPHER: 180 if (csp->csp_cipher_alg != CRYPTO_CHACHA20 && !sc->has_aes) 181 return (EINVAL); 182 if (ossl_lookup_cipher(csp) == NULL) 183 return (EINVAL); 184 break; 185 case CSP_MODE_ETA: 186 if (!sc->has_aes || 187 csp->csp_cipher_alg == CRYPTO_CHACHA20 || 188 ossl_lookup_hash(csp) == NULL || 189 ossl_lookup_cipher(csp) == NULL) 190 return (EINVAL); 191 break; 192 case CSP_MODE_AEAD: 193 switch (csp->csp_cipher_alg) { 194 case CRYPTO_CHACHA20_POLY1305: 195 break; 196 case CRYPTO_AES_NIST_GCM_16: 197 if (!sc->has_aes_gcm || ossl_lookup_cipher(csp) == NULL) 198 return (EINVAL); 199 if (csp->csp_ivlen != AES_GCM_IV_LEN) 200 return (EINVAL); 201 if (csp->csp_auth_mlen != 0 && 202 csp->csp_auth_mlen != GMAC_DIGEST_LEN) 203 return (EINVAL); 204 break; 205 default: 206 return (EINVAL); 207 } 208 break; 209 default: 210 return (EINVAL); 211 } 212 213 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 214 } 215 216 static void 217 ossl_newsession_hash(struct ossl_session *s, 218 const struct crypto_session_params *csp) 219 { 220 struct auth_hash *axf; 221 222 axf = ossl_lookup_hash(csp); 223 s->hash.axf = axf; 224 if (csp->csp_auth_mlen == 0) 225 s->hash.mlen = axf->hashsize; 226 else 227 s->hash.mlen = csp->csp_auth_mlen; 228 229 if (csp->csp_auth_klen == 0) { 230 axf->Init(&s->hash.ictx); 231 } else { 232 if (csp->csp_auth_key != NULL) { 233 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 234 if (axf->Setkey != NULL) { 235 axf->Init(&s->hash.ictx); 236 axf->Setkey(&s->hash.ictx, csp->csp_auth_key, 237 csp->csp_auth_klen); 238 } else { 239 hmac_init_ipad(axf, csp->csp_auth_key, 240 csp->csp_auth_klen, &s->hash.ictx); 241 hmac_init_opad(axf, csp->csp_auth_key, 242 csp->csp_auth_klen, &s->hash.octx); 243 } 244 fpu_kern_leave(curthread, NULL); 245 } 246 } 247 } 248 249 static int 250 ossl_newsession_cipher(struct ossl_session *s, 251 const struct crypto_session_params *csp) 252 { 253 struct ossl_cipher *cipher; 254 int error = 0; 255 256 cipher = ossl_lookup_cipher(csp); 257 if (cipher == NULL) 258 return (EINVAL); 259 260 s->cipher.cipher = cipher; 261 262 if (csp->csp_cipher_key == NULL) 263 return (0); 264 265 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 266 if (cipher->set_encrypt_key != NULL) { 267 error = cipher->set_encrypt_key(csp->csp_cipher_key, 268 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx); 269 if (error != 0) { 270 fpu_kern_leave(curthread, NULL); 271 return (error); 272 } 273 } 274 if (cipher->set_decrypt_key != NULL) 275 error = cipher->set_decrypt_key(csp->csp_cipher_key, 276 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx); 277 fpu_kern_leave(curthread, NULL); 278 279 return (error); 280 } 281 282 static int 283 ossl_newsession(device_t dev, crypto_session_t cses, 284 const struct crypto_session_params *csp) 285 { 286 struct ossl_session *s; 287 int error = 0; 288 289 s = crypto_get_driver_session(cses); 290 switch (csp->csp_mode) { 291 case CSP_MODE_DIGEST: 292 ossl_newsession_hash(s, csp); 293 break; 294 case CSP_MODE_CIPHER: 295 error = ossl_newsession_cipher(s, csp); 296 break; 297 case CSP_MODE_ETA: 298 ossl_newsession_hash(s, csp); 299 error = ossl_newsession_cipher(s, csp); 300 break; 301 case CSP_MODE_AEAD: 302 if (csp->csp_cipher_alg != CRYPTO_CHACHA20_POLY1305) 303 error = ossl_newsession_cipher(s, csp); 304 break; 305 default: 306 __assert_unreachable(); 307 } 308 309 return (error); 310 } 311 312 static int 313 ossl_process_hash(struct ossl_session *s, struct cryptop *crp, 314 const struct crypto_session_params *csp) 315 { 316 struct ossl_hash_context ctx; 317 char digest[HASH_MAX_LEN]; 318 struct auth_hash *axf; 319 int error; 320 321 axf = s->hash.axf; 322 323 if (crp->crp_auth_key == NULL) { 324 ctx = s->hash.ictx; 325 } else { 326 if (axf->Setkey != NULL) { 327 axf->Init(&ctx); 328 axf->Setkey(&ctx, crp->crp_auth_key, 329 csp->csp_auth_klen); 330 } else { 331 hmac_init_ipad(axf, crp->crp_auth_key, 332 csp->csp_auth_klen, &ctx); 333 } 334 } 335 336 if (crp->crp_aad != NULL) 337 error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 338 else 339 error = crypto_apply(crp, crp->crp_aad_start, 340 crp->crp_aad_length, axf->Update, &ctx); 341 if (error) 342 goto out; 343 344 error = crypto_apply(crp, crp->crp_payload_start, 345 crp->crp_payload_length, axf->Update, &ctx); 346 if (error) 347 goto out; 348 349 axf->Final(digest, &ctx); 350 351 if (csp->csp_auth_klen != 0 && axf->Setkey == NULL) { 352 if (crp->crp_auth_key == NULL) 353 ctx = s->hash.octx; 354 else 355 hmac_init_opad(axf, crp->crp_auth_key, 356 csp->csp_auth_klen, &ctx); 357 axf->Update(&ctx, digest, axf->hashsize); 358 axf->Final(digest, &ctx); 359 } 360 361 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 362 char digest2[HASH_MAX_LEN]; 363 364 crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen, 365 digest2); 366 if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0) 367 error = EBADMSG; 368 explicit_bzero(digest2, sizeof(digest2)); 369 } else { 370 crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen, 371 digest); 372 } 373 explicit_bzero(digest, sizeof(digest)); 374 375 out: 376 explicit_bzero(&ctx, sizeof(ctx)); 377 return (error); 378 } 379 380 static int 381 ossl_process_cipher(struct ossl_session *s, struct cryptop *crp, 382 const struct crypto_session_params *csp) 383 { 384 return (s->cipher.cipher->process(&s->cipher, crp, csp)); 385 } 386 387 static int 388 ossl_process_eta(struct ossl_session *s, struct cryptop *crp, 389 const struct crypto_session_params *csp) 390 { 391 int error; 392 393 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 394 error = s->cipher.cipher->process(&s->cipher, crp, csp); 395 if (error == 0) 396 error = ossl_process_hash(s, crp, csp); 397 } else { 398 error = ossl_process_hash(s, crp, csp); 399 if (error == 0) 400 error = s->cipher.cipher->process(&s->cipher, crp, csp); 401 } 402 403 return (error); 404 } 405 406 static int 407 ossl_process_aead(struct ossl_session *s, struct cryptop *crp, 408 const struct crypto_session_params *csp) 409 { 410 if (csp->csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) { 411 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 412 return (ossl_chacha20_poly1305_encrypt(crp, csp)); 413 else 414 return (ossl_chacha20_poly1305_decrypt(crp, csp)); 415 } else { 416 return (s->cipher.cipher->process(&s->cipher, crp, csp)); 417 } 418 } 419 420 static int 421 ossl_process(device_t dev, struct cryptop *crp, int hint) 422 { 423 const struct crypto_session_params *csp; 424 struct ossl_session *s; 425 int error; 426 bool fpu_entered; 427 428 s = crypto_get_driver_session(crp->crp_session); 429 csp = crypto_get_params(crp->crp_session); 430 431 if (is_fpu_kern_thread(0)) { 432 fpu_entered = false; 433 } else { 434 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 435 fpu_entered = true; 436 } 437 438 switch (csp->csp_mode) { 439 case CSP_MODE_DIGEST: 440 error = ossl_process_hash(s, crp, csp); 441 break; 442 case CSP_MODE_CIPHER: 443 error = ossl_process_cipher(s, crp, csp); 444 break; 445 case CSP_MODE_ETA: 446 error = ossl_process_eta(s, crp, csp); 447 break; 448 case CSP_MODE_AEAD: 449 error = ossl_process_aead(s, crp, csp); 450 break; 451 default: 452 __assert_unreachable(); 453 } 454 455 if (fpu_entered) 456 fpu_kern_leave(curthread, NULL); 457 458 crp->crp_etype = error; 459 crypto_done(crp); 460 461 return (0); 462 } 463 464 static device_method_t ossl_methods[] = { 465 DEVMETHOD(device_identify, ossl_identify), 466 DEVMETHOD(device_probe, ossl_probe), 467 DEVMETHOD(device_attach, ossl_attach), 468 DEVMETHOD(device_detach, ossl_detach), 469 470 DEVMETHOD(cryptodev_probesession, ossl_probesession), 471 DEVMETHOD(cryptodev_newsession, ossl_newsession), 472 DEVMETHOD(cryptodev_process, ossl_process), 473 474 DEVMETHOD_END 475 }; 476 477 static driver_t ossl_driver = { 478 "ossl", 479 ossl_methods, 480 sizeof(struct ossl_softc) 481 }; 482 483 DRIVER_MODULE(ossl, nexus, ossl_driver, NULL, NULL); 484 MODULE_VERSION(ossl, 1); 485 MODULE_DEPEND(ossl, crypto, 1, 1, 1); 486