1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020 Netflix, Inc 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any 14 * redistribution must be conditioned upon including a substantially 15 * similar Disclaimer requirement for further binary redistribution. 16 * 17 * NO WARRANTY 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY 21 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, 23 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 26 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 * THE POSSIBILITY OF SUCH DAMAGES. 29 */ 30 31 /* 32 * A driver for the OpenCrypto framework which uses assembly routines 33 * from OpenSSL. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/bus.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 42 #include <machine/fpu.h> 43 44 #include <opencrypto/cryptodev.h> 45 #include <opencrypto/xform_auth.h> 46 47 #include <crypto/openssl/ossl.h> 48 #include <crypto/openssl/ossl_chacha.h> 49 #include <crypto/openssl/ossl_cipher.h> 50 51 #include "cryptodev_if.h" 52 53 static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto"); 54 55 static void 56 ossl_identify(driver_t *driver, device_t parent) 57 { 58 59 if (device_find_child(parent, "ossl", -1) == NULL) 60 BUS_ADD_CHILD(parent, 10, "ossl", DEVICE_UNIT_ANY); 61 } 62 63 static int 64 ossl_probe(device_t dev) 65 { 66 67 device_set_desc(dev, "OpenSSL crypto"); 68 return (BUS_PROBE_DEFAULT); 69 } 70 71 static int 72 ossl_attach(device_t dev) 73 { 74 struct ossl_softc *sc; 75 76 sc = device_get_softc(dev); 77 78 sc->has_aes = sc->has_aes_gcm = false; 79 80 ossl_cpuid(sc); 81 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session), 82 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | 83 CRYPTOCAP_F_ACCEL_SOFTWARE); 84 if (sc->sc_cid < 0) { 85 device_printf(dev, "failed to allocate crypto driver id\n"); 86 return (ENXIO); 87 } 88 89 return (0); 90 } 91 92 static int 93 ossl_detach(device_t dev) 94 { 95 struct ossl_softc *sc; 96 97 sc = device_get_softc(dev); 98 99 crypto_unregister_all(sc->sc_cid); 100 101 return (0); 102 } 103 104 static struct auth_hash * 105 ossl_lookup_hash(const struct crypto_session_params *csp) 106 { 107 108 switch (csp->csp_auth_alg) { 109 case CRYPTO_SHA1: 110 case CRYPTO_SHA1_HMAC: 111 return (&ossl_hash_sha1); 112 case CRYPTO_SHA2_224: 113 case CRYPTO_SHA2_224_HMAC: 114 return (&ossl_hash_sha224); 115 case CRYPTO_SHA2_256: 116 case CRYPTO_SHA2_256_HMAC: 117 return (&ossl_hash_sha256); 118 case CRYPTO_SHA2_384: 119 case CRYPTO_SHA2_384_HMAC: 120 return (&ossl_hash_sha384); 121 case CRYPTO_SHA2_512: 122 case CRYPTO_SHA2_512_HMAC: 123 return (&ossl_hash_sha512); 124 case CRYPTO_POLY1305: 125 return (&ossl_hash_poly1305); 126 default: 127 return (NULL); 128 } 129 } 130 131 static struct ossl_cipher* 132 ossl_lookup_cipher(const struct crypto_session_params *csp) 133 { 134 135 switch (csp->csp_cipher_alg) { 136 case CRYPTO_AES_CBC: 137 switch (csp->csp_cipher_klen * 8) { 138 case 128: 139 case 192: 140 case 256: 141 break; 142 default: 143 return (NULL); 144 } 145 return (&ossl_cipher_aes_cbc); 146 case CRYPTO_AES_NIST_GCM_16: 147 switch (csp->csp_cipher_klen * 8) { 148 case 128: 149 case 192: 150 case 256: 151 break; 152 default: 153 return (NULL); 154 } 155 return (&ossl_cipher_aes_gcm); 156 case CRYPTO_CHACHA20: 157 if (csp->csp_cipher_klen != CHACHA_KEY_SIZE) 158 return (NULL); 159 return (&ossl_cipher_chacha20); 160 default: 161 return (NULL); 162 } 163 } 164 165 static int 166 ossl_probesession(device_t dev, const struct crypto_session_params *csp) 167 { 168 struct ossl_softc *sc = device_get_softc(dev); 169 170 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 171 0) 172 return (EINVAL); 173 switch (csp->csp_mode) { 174 case CSP_MODE_DIGEST: 175 if (ossl_lookup_hash(csp) == NULL) 176 return (EINVAL); 177 break; 178 case CSP_MODE_CIPHER: 179 if (csp->csp_cipher_alg != CRYPTO_CHACHA20 && !sc->has_aes) 180 return (EINVAL); 181 if (ossl_lookup_cipher(csp) == NULL) 182 return (EINVAL); 183 break; 184 case CSP_MODE_ETA: 185 if (!sc->has_aes || 186 csp->csp_cipher_alg == CRYPTO_CHACHA20 || 187 ossl_lookup_hash(csp) == NULL || 188 ossl_lookup_cipher(csp) == NULL) 189 return (EINVAL); 190 break; 191 case CSP_MODE_AEAD: 192 switch (csp->csp_cipher_alg) { 193 case CRYPTO_CHACHA20_POLY1305: 194 break; 195 case CRYPTO_AES_NIST_GCM_16: 196 if (!sc->has_aes_gcm || ossl_lookup_cipher(csp) == NULL) 197 return (EINVAL); 198 if (csp->csp_ivlen != AES_GCM_IV_LEN) 199 return (EINVAL); 200 if (csp->csp_auth_mlen != 0 && 201 csp->csp_auth_mlen != GMAC_DIGEST_LEN) 202 return (EINVAL); 203 break; 204 default: 205 return (EINVAL); 206 } 207 break; 208 default: 209 return (EINVAL); 210 } 211 212 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 213 } 214 215 static void 216 ossl_newsession_hash(struct ossl_session *s, 217 const struct crypto_session_params *csp) 218 { 219 struct auth_hash *axf; 220 221 axf = ossl_lookup_hash(csp); 222 s->hash.axf = axf; 223 if (csp->csp_auth_mlen == 0) 224 s->hash.mlen = axf->hashsize; 225 else 226 s->hash.mlen = csp->csp_auth_mlen; 227 228 if (csp->csp_auth_klen == 0) { 229 axf->Init(&s->hash.ictx); 230 } else { 231 if (csp->csp_auth_key != NULL) { 232 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 233 if (axf->Setkey != NULL) { 234 axf->Init(&s->hash.ictx); 235 axf->Setkey(&s->hash.ictx, csp->csp_auth_key, 236 csp->csp_auth_klen); 237 } else { 238 hmac_init_ipad(axf, csp->csp_auth_key, 239 csp->csp_auth_klen, &s->hash.ictx); 240 hmac_init_opad(axf, csp->csp_auth_key, 241 csp->csp_auth_klen, &s->hash.octx); 242 } 243 fpu_kern_leave(curthread, NULL); 244 } 245 } 246 } 247 248 static int 249 ossl_newsession_cipher(struct ossl_session *s, 250 const struct crypto_session_params *csp) 251 { 252 struct ossl_cipher *cipher; 253 int error = 0; 254 255 cipher = ossl_lookup_cipher(csp); 256 if (cipher == NULL) 257 return (EINVAL); 258 259 s->cipher.cipher = cipher; 260 261 if (csp->csp_cipher_key == NULL) 262 return (0); 263 264 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 265 if (cipher->set_encrypt_key != NULL) { 266 error = cipher->set_encrypt_key(csp->csp_cipher_key, 267 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx); 268 if (error != 0) { 269 fpu_kern_leave(curthread, NULL); 270 return (error); 271 } 272 } 273 if (cipher->set_decrypt_key != NULL) 274 error = cipher->set_decrypt_key(csp->csp_cipher_key, 275 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx); 276 fpu_kern_leave(curthread, NULL); 277 278 return (error); 279 } 280 281 static int 282 ossl_newsession(device_t dev, crypto_session_t cses, 283 const struct crypto_session_params *csp) 284 { 285 struct ossl_session *s; 286 int error = 0; 287 288 s = crypto_get_driver_session(cses); 289 switch (csp->csp_mode) { 290 case CSP_MODE_DIGEST: 291 ossl_newsession_hash(s, csp); 292 break; 293 case CSP_MODE_CIPHER: 294 error = ossl_newsession_cipher(s, csp); 295 break; 296 case CSP_MODE_ETA: 297 ossl_newsession_hash(s, csp); 298 error = ossl_newsession_cipher(s, csp); 299 break; 300 case CSP_MODE_AEAD: 301 if (csp->csp_cipher_alg != CRYPTO_CHACHA20_POLY1305) 302 error = ossl_newsession_cipher(s, csp); 303 break; 304 default: 305 __assert_unreachable(); 306 } 307 308 return (error); 309 } 310 311 static int 312 ossl_process_hash(struct ossl_session *s, struct cryptop *crp, 313 const struct crypto_session_params *csp) 314 { 315 struct ossl_hash_context ctx; 316 char digest[HASH_MAX_LEN]; 317 struct auth_hash *axf; 318 int error; 319 320 axf = s->hash.axf; 321 322 if (crp->crp_auth_key == NULL) { 323 ctx = s->hash.ictx; 324 } else { 325 if (axf->Setkey != NULL) { 326 axf->Init(&ctx); 327 axf->Setkey(&ctx, crp->crp_auth_key, 328 csp->csp_auth_klen); 329 } else { 330 hmac_init_ipad(axf, crp->crp_auth_key, 331 csp->csp_auth_klen, &ctx); 332 } 333 } 334 335 if (crp->crp_aad != NULL) 336 error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 337 else 338 error = crypto_apply(crp, crp->crp_aad_start, 339 crp->crp_aad_length, axf->Update, &ctx); 340 if (error) 341 goto out; 342 343 error = crypto_apply(crp, crp->crp_payload_start, 344 crp->crp_payload_length, axf->Update, &ctx); 345 if (error) 346 goto out; 347 348 axf->Final(digest, &ctx); 349 350 if (csp->csp_auth_klen != 0 && axf->Setkey == NULL) { 351 if (crp->crp_auth_key == NULL) 352 ctx = s->hash.octx; 353 else 354 hmac_init_opad(axf, crp->crp_auth_key, 355 csp->csp_auth_klen, &ctx); 356 axf->Update(&ctx, digest, axf->hashsize); 357 axf->Final(digest, &ctx); 358 } 359 360 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 361 char digest2[HASH_MAX_LEN]; 362 363 crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen, 364 digest2); 365 if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0) 366 error = EBADMSG; 367 explicit_bzero(digest2, sizeof(digest2)); 368 } else { 369 crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen, 370 digest); 371 } 372 explicit_bzero(digest, sizeof(digest)); 373 374 out: 375 explicit_bzero(&ctx, sizeof(ctx)); 376 return (error); 377 } 378 379 static int 380 ossl_process_cipher(struct ossl_session *s, struct cryptop *crp, 381 const struct crypto_session_params *csp) 382 { 383 return (s->cipher.cipher->process(&s->cipher, crp, csp)); 384 } 385 386 static int 387 ossl_process_eta(struct ossl_session *s, struct cryptop *crp, 388 const struct crypto_session_params *csp) 389 { 390 int error; 391 392 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 393 error = s->cipher.cipher->process(&s->cipher, crp, csp); 394 if (error == 0) 395 error = ossl_process_hash(s, crp, csp); 396 } else { 397 error = ossl_process_hash(s, crp, csp); 398 if (error == 0) 399 error = s->cipher.cipher->process(&s->cipher, crp, csp); 400 } 401 402 return (error); 403 } 404 405 static int 406 ossl_process_aead(struct ossl_session *s, struct cryptop *crp, 407 const struct crypto_session_params *csp) 408 { 409 if (csp->csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) { 410 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 411 return (ossl_chacha20_poly1305_encrypt(crp, csp)); 412 else 413 return (ossl_chacha20_poly1305_decrypt(crp, csp)); 414 } else { 415 return (s->cipher.cipher->process(&s->cipher, crp, csp)); 416 } 417 } 418 419 static int 420 ossl_process(device_t dev, struct cryptop *crp, int hint) 421 { 422 const struct crypto_session_params *csp; 423 struct ossl_session *s; 424 int error; 425 bool fpu_entered; 426 427 s = crypto_get_driver_session(crp->crp_session); 428 csp = crypto_get_params(crp->crp_session); 429 430 if (is_fpu_kern_thread(0)) { 431 fpu_entered = false; 432 } else { 433 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); 434 fpu_entered = true; 435 } 436 437 switch (csp->csp_mode) { 438 case CSP_MODE_DIGEST: 439 error = ossl_process_hash(s, crp, csp); 440 break; 441 case CSP_MODE_CIPHER: 442 error = ossl_process_cipher(s, crp, csp); 443 break; 444 case CSP_MODE_ETA: 445 error = ossl_process_eta(s, crp, csp); 446 break; 447 case CSP_MODE_AEAD: 448 error = ossl_process_aead(s, crp, csp); 449 break; 450 default: 451 __assert_unreachable(); 452 } 453 454 if (fpu_entered) 455 fpu_kern_leave(curthread, NULL); 456 457 crp->crp_etype = error; 458 crypto_done(crp); 459 460 return (0); 461 } 462 463 static device_method_t ossl_methods[] = { 464 DEVMETHOD(device_identify, ossl_identify), 465 DEVMETHOD(device_probe, ossl_probe), 466 DEVMETHOD(device_attach, ossl_attach), 467 DEVMETHOD(device_detach, ossl_detach), 468 469 DEVMETHOD(cryptodev_probesession, ossl_probesession), 470 DEVMETHOD(cryptodev_newsession, ossl_newsession), 471 DEVMETHOD(cryptodev_process, ossl_process), 472 473 DEVMETHOD_END 474 }; 475 476 static driver_t ossl_driver = { 477 "ossl", 478 ossl_methods, 479 sizeof(struct ossl_softc) 480 }; 481 482 DRIVER_MODULE(ossl, nexus, ossl_driver, NULL, NULL); 483 MODULE_VERSION(ossl, 1); 484 MODULE_DEPEND(ossl, crypto, 1, 1, 1); 485