1 /*- 2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4 * Copyright (c) 2014,2016 The FreeBSD Foundation 5 * Copyright (c) 2020 Ampere Computing 6 * All rights reserved. 7 * 8 * Portions of this software were developed by John-Mark Gurney 9 * under sponsorship of the FreeBSD Foundation and 10 * Rubicon Communications, LLC (Netgate). 11 * 12 * This software was developed by Andrew Turner under 13 * sponsorship from the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 /* 38 * This is based on the aesni code. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/kernel.h> 47 #include <sys/bus.h> 48 #include <sys/endian.h> 49 #include <sys/lock.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/module.h> 53 #include <sys/mutex.h> 54 #include <sys/queue.h> 55 #include <sys/rwlock.h> 56 #include <sys/smp.h> 57 #include <sys/uio.h> 58 59 #include <machine/vfp.h> 60 61 #include <opencrypto/cryptodev.h> 62 #include <opencrypto/gmac.h> 63 #include <cryptodev_if.h> 64 #include <crypto/armv8/armv8_crypto.h> 65 #include <crypto/rijndael/rijndael.h> 66 67 struct armv8_crypto_softc { 68 int dieing; 69 int32_t cid; 70 struct rwlock lock; 71 bool has_pmul; 72 }; 73 74 static struct mtx *ctx_mtx; 75 static struct fpu_kern_ctx **ctx_vfp; 76 77 #define AQUIRE_CTX(i, ctx) \ 78 do { \ 79 (i) = PCPU_GET(cpuid); \ 80 mtx_lock(&ctx_mtx[(i)]); \ 81 (ctx) = ctx_vfp[(i)]; \ 82 } while (0) 83 #define RELEASE_CTX(i, ctx) \ 84 do { \ 85 mtx_unlock(&ctx_mtx[(i)]); \ 86 (i) = -1; \ 87 (ctx) = NULL; \ 88 } while (0) 89 90 static int armv8_crypto_cipher_process(struct armv8_crypto_session *, 91 struct cryptop *); 92 93 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data"); 94 95 static void 96 armv8_crypto_identify(driver_t *drv, device_t parent) 97 { 98 99 /* NB: order 10 is so we get attached after h/w devices */ 100 if (device_find_child(parent, "armv8crypto", -1) == NULL && 101 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0) 102 panic("ARMv8 crypto: could not attach"); 103 } 104 105 static int 106 armv8_crypto_probe(device_t dev) 107 { 108 uint64_t reg; 109 int ret = ENXIO; 110 111 reg = READ_SPECIALREG(id_aa64isar0_el1); 112 113 switch (ID_AA64ISAR0_AES_VAL(reg)) { 114 case ID_AA64ISAR0_AES_BASE: 115 ret = 0; 116 device_set_desc(dev, "AES-CBC,AES-XTS"); 117 break; 118 case ID_AA64ISAR0_AES_PMULL: 119 ret = 0; 120 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM"); 121 break; 122 default: 123 break; 124 case ID_AA64ISAR0_AES_NONE: 125 device_printf(dev, "CPU lacks AES instructions\n"); 126 break; 127 } 128 129 /* TODO: Check more fields as we support more features */ 130 131 return (ret); 132 } 133 134 static int 135 armv8_crypto_attach(device_t dev) 136 { 137 struct armv8_crypto_softc *sc; 138 uint64_t reg; 139 int i; 140 141 sc = device_get_softc(dev); 142 sc->dieing = 0; 143 144 reg = READ_SPECIALREG(id_aa64isar0_el1); 145 146 if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL) 147 sc->has_pmul = true; 148 149 sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session), 150 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); 151 if (sc->cid < 0) { 152 device_printf(dev, "Could not get crypto driver id.\n"); 153 return (ENOMEM); 154 } 155 156 rw_init(&sc->lock, "armv8crypto"); 157 158 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO, 159 M_WAITOK|M_ZERO); 160 ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO, 161 M_WAITOK|M_ZERO); 162 163 CPU_FOREACH(i) { 164 ctx_vfp[i] = fpu_kern_alloc_ctx(0); 165 mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW); 166 } 167 168 return (0); 169 } 170 171 static int 172 armv8_crypto_detach(device_t dev) 173 { 174 struct armv8_crypto_softc *sc; 175 int i; 176 177 sc = device_get_softc(dev); 178 179 rw_wlock(&sc->lock); 180 sc->dieing = 1; 181 rw_wunlock(&sc->lock); 182 crypto_unregister_all(sc->cid); 183 184 rw_destroy(&sc->lock); 185 186 CPU_FOREACH(i) { 187 if (ctx_vfp[i] != NULL) { 188 mtx_destroy(&ctx_mtx[i]); 189 fpu_kern_free_ctx(ctx_vfp[i]); 190 } 191 ctx_vfp[i] = NULL; 192 } 193 free(ctx_mtx, M_ARMV8_CRYPTO); 194 ctx_mtx = NULL; 195 free(ctx_vfp, M_ARMV8_CRYPTO); 196 ctx_vfp = NULL; 197 198 return (0); 199 } 200 201 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD) 202 203 static int 204 armv8_crypto_probesession(device_t dev, 205 const struct crypto_session_params *csp) 206 { 207 struct armv8_crypto_softc *sc; 208 209 sc = device_get_softc(dev); 210 211 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 212 return (EINVAL); 213 214 switch (csp->csp_mode) { 215 case CSP_MODE_AEAD: 216 switch (csp->csp_cipher_alg) { 217 case CRYPTO_AES_NIST_GCM_16: 218 if (!sc->has_pmul) 219 return (EINVAL); 220 if (csp->csp_auth_mlen != 0 && 221 csp->csp_auth_mlen != GMAC_DIGEST_LEN) 222 return (EINVAL); 223 switch (csp->csp_cipher_klen * 8) { 224 case 128: 225 case 192: 226 case 256: 227 break; 228 default: 229 return (EINVAL); 230 } 231 break; 232 default: 233 return (EINVAL); 234 } 235 break; 236 case CSP_MODE_CIPHER: 237 switch (csp->csp_cipher_alg) { 238 case CRYPTO_AES_CBC: 239 if (csp->csp_ivlen != AES_BLOCK_LEN) 240 return (EINVAL); 241 switch (csp->csp_cipher_klen * 8) { 242 case 128: 243 case 192: 244 case 256: 245 break; 246 default: 247 return (EINVAL); 248 } 249 break; 250 case CRYPTO_AES_XTS: 251 if (csp->csp_ivlen != AES_XTS_IV_LEN) 252 return (EINVAL); 253 switch (csp->csp_cipher_klen * 8) { 254 case 256: 255 case 512: 256 break; 257 default: 258 return (EINVAL); 259 } 260 break; 261 default: 262 return (EINVAL); 263 } 264 break; 265 default: 266 return (EINVAL); 267 } 268 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); 269 } 270 271 static int 272 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, 273 const struct crypto_session_params *csp, const uint8_t *key, int keylen) 274 { 275 __uint128_val_t H; 276 struct fpu_kern_ctx *ctx; 277 int kt, i; 278 279 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 280 keylen /= 2; 281 282 switch (keylen * 8) { 283 case 128: 284 case 192: 285 case 256: 286 break; 287 default: 288 return (EINVAL); 289 } 290 291 kt = is_fpu_kern_thread(0); 292 if (!kt) { 293 AQUIRE_CTX(i, ctx); 294 fpu_kern_enter(curthread, ctx, 295 FPU_KERN_NORMAL | FPU_KERN_KTHR); 296 } 297 298 aes_v8_set_encrypt_key(key, 299 keylen * 8, &ses->enc_schedule); 300 301 if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) || 302 (csp->csp_cipher_alg == CRYPTO_AES_CBC)) 303 aes_v8_set_decrypt_key(key, 304 keylen * 8, &ses->dec_schedule); 305 306 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 307 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule); 308 309 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { 310 memset(H.c, 0, sizeof(H.c)); 311 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule); 312 H.u[0] = bswap64(H.u[0]); 313 H.u[1] = bswap64(H.u[1]); 314 gcm_init_v8(ses->Htable, H.u); 315 } 316 317 if (!kt) { 318 fpu_kern_leave(curthread, ctx); 319 RELEASE_CTX(i, ctx); 320 } 321 322 return (0); 323 } 324 325 static int 326 armv8_crypto_newsession(device_t dev, crypto_session_t cses, 327 const struct crypto_session_params *csp) 328 { 329 struct armv8_crypto_softc *sc; 330 struct armv8_crypto_session *ses; 331 int error; 332 333 sc = device_get_softc(dev); 334 rw_wlock(&sc->lock); 335 if (sc->dieing) { 336 rw_wunlock(&sc->lock); 337 return (EINVAL); 338 } 339 340 ses = crypto_get_driver_session(cses); 341 error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, 342 csp->csp_cipher_klen); 343 rw_wunlock(&sc->lock); 344 return (error); 345 } 346 347 static int 348 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused) 349 { 350 struct armv8_crypto_session *ses; 351 352 ses = crypto_get_driver_session(crp->crp_session); 353 crp->crp_etype = armv8_crypto_cipher_process(ses, crp); 354 crypto_done(crp); 355 return (0); 356 } 357 358 static uint8_t * 359 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated) 360 { 361 uint8_t *addr; 362 363 addr = crypto_contiguous_subsegment(crp, start, length); 364 if (addr != NULL) { 365 *allocated = 0; 366 return (addr); 367 } 368 addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT); 369 if (addr != NULL) { 370 *allocated = 1; 371 crypto_copydata(crp, start, length, addr); 372 } else 373 *allocated = 0; 374 return (addr); 375 } 376 377 static int 378 armv8_crypto_cipher_process(struct armv8_crypto_session *ses, 379 struct cryptop *crp) 380 { 381 const struct crypto_session_params *csp; 382 struct fpu_kern_ctx *ctx; 383 uint8_t *buf, *authbuf, *outbuf; 384 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN]; 385 int allocated, authallocated, outallocated, i; 386 int encflag; 387 int kt; 388 int error; 389 bool outcopy; 390 391 csp = crypto_get_params(crp->crp_session); 392 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 393 394 allocated = 0; 395 outallocated = 0; 396 authallocated = 0; 397 authbuf = NULL; 398 kt = 1; 399 400 buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start, 401 crp->crp_payload_length, &allocated); 402 if (buf == NULL) 403 return (ENOMEM); 404 405 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { 406 if (crp->crp_aad != NULL) 407 authbuf = crp->crp_aad; 408 else 409 authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start, 410 crp->crp_aad_length, &authallocated); 411 if (authbuf == NULL) { 412 error = ENOMEM; 413 goto out; 414 } 415 } 416 417 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 418 outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf, 419 crp->crp_payload_output_start, crp->crp_payload_length); 420 if (outbuf == NULL) { 421 outcopy = true; 422 if (allocated) 423 outbuf = buf; 424 else { 425 outbuf = malloc(crp->crp_payload_length, 426 M_ARMV8_CRYPTO, M_NOWAIT); 427 if (outbuf == NULL) { 428 error = ENOMEM; 429 goto out; 430 } 431 outallocated = true; 432 } 433 } else 434 outcopy = false; 435 } else { 436 outbuf = buf; 437 outcopy = allocated; 438 } 439 440 kt = is_fpu_kern_thread(0); 441 if (!kt) { 442 AQUIRE_CTX(i, ctx); 443 fpu_kern_enter(curthread, ctx, 444 FPU_KERN_NORMAL | FPU_KERN_KTHR); 445 } 446 447 if (crp->crp_cipher_key != NULL) { 448 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key, 449 csp->csp_cipher_klen); 450 } 451 452 crypto_read_iv(crp, iv); 453 454 /* Do work */ 455 switch (csp->csp_cipher_alg) { 456 case CRYPTO_AES_CBC: 457 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { 458 error = EINVAL; 459 goto out; 460 } 461 if (encflag) 462 armv8_aes_encrypt_cbc(&ses->enc_schedule, 463 crp->crp_payload_length, buf, buf, iv); 464 else 465 armv8_aes_decrypt_cbc(&ses->dec_schedule, 466 crp->crp_payload_length, buf, iv); 467 break; 468 case CRYPTO_AES_XTS: 469 if (encflag) 470 armv8_aes_encrypt_xts(&ses->enc_schedule, 471 &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, 472 buf, iv); 473 else 474 armv8_aes_decrypt_xts(&ses->dec_schedule, 475 &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, 476 buf, iv); 477 break; 478 case CRYPTO_AES_NIST_GCM_16: 479 if (encflag) { 480 memset(tag, 0, sizeof(tag)); 481 armv8_aes_encrypt_gcm(&ses->enc_schedule, 482 crp->crp_payload_length, 483 buf, outbuf, 484 crp->crp_aad_length, authbuf, 485 tag, iv, ses->Htable); 486 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag), 487 tag); 488 } else { 489 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag), 490 tag); 491 if (armv8_aes_decrypt_gcm(&ses->enc_schedule, 492 crp->crp_payload_length, 493 buf, outbuf, 494 crp->crp_aad_length, authbuf, 495 tag, iv, ses->Htable) != 0) { 496 error = EBADMSG; 497 goto out; 498 } 499 } 500 break; 501 } 502 503 if (outcopy) 504 crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ? 505 crp->crp_payload_output_start : crp->crp_payload_start, 506 crp->crp_payload_length, outbuf); 507 508 error = 0; 509 out: 510 if (!kt) { 511 fpu_kern_leave(curthread, ctx); 512 RELEASE_CTX(i, ctx); 513 } 514 515 if (allocated) 516 zfree(buf, M_ARMV8_CRYPTO); 517 if (authallocated) 518 zfree(authbuf, M_ARMV8_CRYPTO); 519 if (outallocated) 520 zfree(outbuf, M_ARMV8_CRYPTO); 521 explicit_bzero(iv, sizeof(iv)); 522 explicit_bzero(tag, sizeof(tag)); 523 524 return (error); 525 } 526 527 static device_method_t armv8_crypto_methods[] = { 528 DEVMETHOD(device_identify, armv8_crypto_identify), 529 DEVMETHOD(device_probe, armv8_crypto_probe), 530 DEVMETHOD(device_attach, armv8_crypto_attach), 531 DEVMETHOD(device_detach, armv8_crypto_detach), 532 533 DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession), 534 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession), 535 DEVMETHOD(cryptodev_process, armv8_crypto_process), 536 537 DEVMETHOD_END, 538 }; 539 540 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods, 541 sizeof(struct armv8_crypto_softc)); 542 static devclass_t armv8_crypto_devclass; 543 544 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass, 545 0, 0); 546