1 /*- 2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4 * Copyright (c) 2014,2016 The FreeBSD Foundation 5 * Copyright (c) 2020 Ampere Computing 6 * All rights reserved. 7 * 8 * Portions of this software were developed by John-Mark Gurney 9 * under sponsorship of the FreeBSD Foundation and 10 * Rubicon Communications, LLC (Netgate). 11 * 12 * This software was developed by Andrew Turner under 13 * sponsorship from the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 /* 38 * This is based on the aesni code. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/kernel.h> 47 #include <sys/bus.h> 48 #include <sys/endian.h> 49 #include <sys/lock.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/module.h> 53 #include <sys/mutex.h> 54 #include <sys/queue.h> 55 #include <sys/rwlock.h> 56 #include <sys/smp.h> 57 #include <sys/uio.h> 58 59 #include <machine/vfp.h> 60 61 #include <opencrypto/cryptodev.h> 62 #include <opencrypto/gmac.h> 63 #include <cryptodev_if.h> 64 #include <crypto/armv8/armv8_crypto.h> 65 #include <crypto/rijndael/rijndael.h> 66 67 struct armv8_crypto_softc { 68 int dieing; 69 int32_t cid; 70 struct rwlock lock; 71 bool has_pmul; 72 }; 73 74 static struct mtx *ctx_mtx; 75 static struct fpu_kern_ctx **ctx_vfp; 76 77 #define AQUIRE_CTX(i, ctx) \ 78 do { \ 79 (i) = PCPU_GET(cpuid); \ 80 mtx_lock(&ctx_mtx[(i)]); \ 81 (ctx) = ctx_vfp[(i)]; \ 82 } while (0) 83 #define RELEASE_CTX(i, ctx) \ 84 do { \ 85 mtx_unlock(&ctx_mtx[(i)]); \ 86 (i) = -1; \ 87 (ctx) = NULL; \ 88 } while (0) 89 90 static int armv8_crypto_cipher_process(struct armv8_crypto_session *, 91 struct cryptop *); 92 93 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data"); 94 95 static void 96 armv8_crypto_identify(driver_t *drv, device_t parent) 97 { 98 99 /* NB: order 10 is so we get attached after h/w devices */ 100 if (device_find_child(parent, "armv8crypto", -1) == NULL && 101 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0) 102 panic("ARMv8 crypto: could not attach"); 103 } 104 105 static int 106 armv8_crypto_probe(device_t dev) 107 { 108 uint64_t reg; 109 int ret = ENXIO; 110 111 reg = READ_SPECIALREG(id_aa64isar0_el1); 112 113 switch (ID_AA64ISAR0_AES_VAL(reg)) { 114 case ID_AA64ISAR0_AES_BASE: 115 ret = 0; 116 device_set_desc(dev, "AES-CBC,AES-XTS"); 117 break; 118 case ID_AA64ISAR0_AES_PMULL: 119 ret = 0; 120 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM"); 121 break; 122 default: 123 break; 124 case ID_AA64ISAR0_AES_NONE: 125 device_printf(dev, "CPU lacks AES instructions\n"); 126 break; 127 } 128 129 /* TODO: Check more fields as we support more features */ 130 131 return (ret); 132 } 133 134 static int 135 armv8_crypto_attach(device_t dev) 136 { 137 struct armv8_crypto_softc *sc; 138 uint64_t reg; 139 int i; 140 141 sc = device_get_softc(dev); 142 sc->dieing = 0; 143 144 reg = READ_SPECIALREG(id_aa64isar0_el1); 145 146 if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL) 147 sc->has_pmul = true; 148 149 sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session), 150 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); 151 if (sc->cid < 0) { 152 device_printf(dev, "Could not get crypto driver id.\n"); 153 return (ENOMEM); 154 } 155 156 rw_init(&sc->lock, "armv8crypto"); 157 158 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO, 159 M_WAITOK|M_ZERO); 160 ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO, 161 M_WAITOK|M_ZERO); 162 163 CPU_FOREACH(i) { 164 ctx_vfp[i] = fpu_kern_alloc_ctx(0); 165 mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW); 166 } 167 168 return (0); 169 } 170 171 static int 172 armv8_crypto_detach(device_t dev) 173 { 174 struct armv8_crypto_softc *sc; 175 int i; 176 177 sc = device_get_softc(dev); 178 179 rw_wlock(&sc->lock); 180 sc->dieing = 1; 181 rw_wunlock(&sc->lock); 182 crypto_unregister_all(sc->cid); 183 184 rw_destroy(&sc->lock); 185 186 CPU_FOREACH(i) { 187 if (ctx_vfp[i] != NULL) { 188 mtx_destroy(&ctx_mtx[i]); 189 fpu_kern_free_ctx(ctx_vfp[i]); 190 } 191 ctx_vfp[i] = NULL; 192 } 193 free(ctx_mtx, M_ARMV8_CRYPTO); 194 ctx_mtx = NULL; 195 free(ctx_vfp, M_ARMV8_CRYPTO); 196 ctx_vfp = NULL; 197 198 return (0); 199 } 200 201 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD) 202 203 static int 204 armv8_crypto_probesession(device_t dev, 205 const struct crypto_session_params *csp) 206 { 207 struct armv8_crypto_softc *sc; 208 209 sc = device_get_softc(dev); 210 211 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 212 return (EINVAL); 213 214 switch (csp->csp_mode) { 215 case CSP_MODE_AEAD: 216 switch (csp->csp_cipher_alg) { 217 case CRYPTO_AES_NIST_GCM_16: 218 if (!sc->has_pmul) 219 return (EINVAL); 220 if (csp->csp_ivlen != AES_GCM_IV_LEN) 221 return (EINVAL); 222 if (csp->csp_auth_mlen != 0 && 223 csp->csp_auth_mlen != GMAC_DIGEST_LEN) 224 return (EINVAL); 225 switch (csp->csp_cipher_klen * 8) { 226 case 128: 227 case 192: 228 case 256: 229 break; 230 default: 231 return (EINVAL); 232 } 233 break; 234 default: 235 return (EINVAL); 236 } 237 break; 238 case CSP_MODE_CIPHER: 239 switch (csp->csp_cipher_alg) { 240 case CRYPTO_AES_CBC: 241 if (csp->csp_ivlen != AES_BLOCK_LEN) 242 return (EINVAL); 243 switch (csp->csp_cipher_klen * 8) { 244 case 128: 245 case 192: 246 case 256: 247 break; 248 default: 249 return (EINVAL); 250 } 251 break; 252 case CRYPTO_AES_XTS: 253 if (csp->csp_ivlen != AES_XTS_IV_LEN) 254 return (EINVAL); 255 switch (csp->csp_cipher_klen * 8) { 256 case 256: 257 case 512: 258 break; 259 default: 260 return (EINVAL); 261 } 262 break; 263 default: 264 return (EINVAL); 265 } 266 break; 267 default: 268 return (EINVAL); 269 } 270 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 271 } 272 273 static int 274 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, 275 const struct crypto_session_params *csp, const uint8_t *key, int keylen) 276 { 277 __uint128_val_t H; 278 struct fpu_kern_ctx *ctx; 279 int kt, i; 280 281 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 282 keylen /= 2; 283 284 switch (keylen * 8) { 285 case 128: 286 case 192: 287 case 256: 288 break; 289 default: 290 return (EINVAL); 291 } 292 293 kt = is_fpu_kern_thread(0); 294 if (!kt) { 295 AQUIRE_CTX(i, ctx); 296 fpu_kern_enter(curthread, ctx, 297 FPU_KERN_NORMAL | FPU_KERN_KTHR); 298 } 299 300 aes_v8_set_encrypt_key(key, 301 keylen * 8, &ses->enc_schedule); 302 303 if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) || 304 (csp->csp_cipher_alg == CRYPTO_AES_CBC)) 305 aes_v8_set_decrypt_key(key, 306 keylen * 8, &ses->dec_schedule); 307 308 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 309 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule); 310 311 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { 312 memset(H.c, 0, sizeof(H.c)); 313 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule); 314 H.u[0] = bswap64(H.u[0]); 315 H.u[1] = bswap64(H.u[1]); 316 gcm_init_v8(ses->Htable, H.u); 317 } 318 319 if (!kt) { 320 fpu_kern_leave(curthread, ctx); 321 RELEASE_CTX(i, ctx); 322 } 323 324 return (0); 325 } 326 327 static int 328 armv8_crypto_newsession(device_t dev, crypto_session_t cses, 329 const struct crypto_session_params *csp) 330 { 331 struct armv8_crypto_softc *sc; 332 struct armv8_crypto_session *ses; 333 int error; 334 335 sc = device_get_softc(dev); 336 rw_wlock(&sc->lock); 337 if (sc->dieing) { 338 rw_wunlock(&sc->lock); 339 return (EINVAL); 340 } 341 342 ses = crypto_get_driver_session(cses); 343 error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, 344 csp->csp_cipher_klen); 345 rw_wunlock(&sc->lock); 346 return (error); 347 } 348 349 static int 350 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused) 351 { 352 struct armv8_crypto_session *ses; 353 354 ses = crypto_get_driver_session(crp->crp_session); 355 crp->crp_etype = armv8_crypto_cipher_process(ses, crp); 356 crypto_done(crp); 357 return (0); 358 } 359 360 static uint8_t * 361 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated) 362 { 363 uint8_t *addr; 364 365 addr = crypto_contiguous_subsegment(crp, start, length); 366 if (addr != NULL) { 367 *allocated = 0; 368 return (addr); 369 } 370 addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT); 371 if (addr != NULL) { 372 *allocated = 1; 373 crypto_copydata(crp, start, length, addr); 374 } else 375 *allocated = 0; 376 return (addr); 377 } 378 379 static int 380 armv8_crypto_cipher_process(struct armv8_crypto_session *ses, 381 struct cryptop *crp) 382 { 383 const struct crypto_session_params *csp; 384 struct fpu_kern_ctx *ctx; 385 uint8_t *buf, *authbuf, *outbuf; 386 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN]; 387 int allocated, authallocated, outallocated, i; 388 int encflag; 389 int kt; 390 int error; 391 bool outcopy; 392 393 csp = crypto_get_params(crp->crp_session); 394 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 395 396 allocated = 0; 397 outallocated = 0; 398 authallocated = 0; 399 authbuf = NULL; 400 kt = 1; 401 402 buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start, 403 crp->crp_payload_length, &allocated); 404 if (buf == NULL) 405 return (ENOMEM); 406 407 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { 408 if (crp->crp_aad != NULL) 409 authbuf = crp->crp_aad; 410 else 411 authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start, 412 crp->crp_aad_length, &authallocated); 413 if (authbuf == NULL) { 414 error = ENOMEM; 415 goto out; 416 } 417 } 418 419 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 420 outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf, 421 crp->crp_payload_output_start, crp->crp_payload_length); 422 if (outbuf == NULL) { 423 outcopy = true; 424 if (allocated) 425 outbuf = buf; 426 else { 427 outbuf = malloc(crp->crp_payload_length, 428 M_ARMV8_CRYPTO, M_NOWAIT); 429 if (outbuf == NULL) { 430 error = ENOMEM; 431 goto out; 432 } 433 outallocated = true; 434 } 435 } else 436 outcopy = false; 437 } else { 438 outbuf = buf; 439 outcopy = allocated; 440 } 441 442 kt = is_fpu_kern_thread(0); 443 if (!kt) { 444 AQUIRE_CTX(i, ctx); 445 fpu_kern_enter(curthread, ctx, 446 FPU_KERN_NORMAL | FPU_KERN_KTHR); 447 } 448 449 if (crp->crp_cipher_key != NULL) { 450 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key, 451 csp->csp_cipher_klen); 452 } 453 454 crypto_read_iv(crp, iv); 455 456 /* Do work */ 457 switch (csp->csp_cipher_alg) { 458 case CRYPTO_AES_CBC: 459 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { 460 error = EINVAL; 461 goto out; 462 } 463 if (encflag) 464 armv8_aes_encrypt_cbc(&ses->enc_schedule, 465 crp->crp_payload_length, buf, buf, iv); 466 else 467 armv8_aes_decrypt_cbc(&ses->dec_schedule, 468 crp->crp_payload_length, buf, iv); 469 break; 470 case CRYPTO_AES_XTS: 471 if (encflag) 472 armv8_aes_encrypt_xts(&ses->enc_schedule, 473 &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, 474 buf, iv); 475 else 476 armv8_aes_decrypt_xts(&ses->dec_schedule, 477 &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, 478 buf, iv); 479 break; 480 case CRYPTO_AES_NIST_GCM_16: 481 if (encflag) { 482 memset(tag, 0, sizeof(tag)); 483 armv8_aes_encrypt_gcm(&ses->enc_schedule, 484 crp->crp_payload_length, 485 buf, outbuf, 486 crp->crp_aad_length, authbuf, 487 tag, iv, ses->Htable); 488 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag), 489 tag); 490 } else { 491 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag), 492 tag); 493 if (armv8_aes_decrypt_gcm(&ses->enc_schedule, 494 crp->crp_payload_length, 495 buf, outbuf, 496 crp->crp_aad_length, authbuf, 497 tag, iv, ses->Htable) != 0) { 498 error = EBADMSG; 499 goto out; 500 } 501 } 502 break; 503 } 504 505 if (outcopy) 506 crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ? 507 crp->crp_payload_output_start : crp->crp_payload_start, 508 crp->crp_payload_length, outbuf); 509 510 error = 0; 511 out: 512 if (!kt) { 513 fpu_kern_leave(curthread, ctx); 514 RELEASE_CTX(i, ctx); 515 } 516 517 if (allocated) 518 zfree(buf, M_ARMV8_CRYPTO); 519 if (authallocated) 520 zfree(authbuf, M_ARMV8_CRYPTO); 521 if (outallocated) 522 zfree(outbuf, M_ARMV8_CRYPTO); 523 explicit_bzero(iv, sizeof(iv)); 524 explicit_bzero(tag, sizeof(tag)); 525 526 return (error); 527 } 528 529 static device_method_t armv8_crypto_methods[] = { 530 DEVMETHOD(device_identify, armv8_crypto_identify), 531 DEVMETHOD(device_probe, armv8_crypto_probe), 532 DEVMETHOD(device_attach, armv8_crypto_attach), 533 DEVMETHOD(device_detach, armv8_crypto_detach), 534 535 DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession), 536 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession), 537 DEVMETHOD(cryptodev_process, armv8_crypto_process), 538 539 DEVMETHOD_END, 540 }; 541 542 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods, 543 sizeof(struct armv8_crypto_softc)); 544 static devclass_t armv8_crypto_devclass; 545 546 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass, 547 0, 0); 548