1 /*- 2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4 * Copyright (c) 2014,2016 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * Portions of this software were developed by John-Mark Gurney 8 * under sponsorship of the FreeBSD Foundation and 9 * Rubicon Communications, LLC (Netgate). 10 * 11 * This software was developed by Andrew Turner under 12 * sponsorship from the FreeBSD Foundation. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 */ 35 36 /* 37 * This is based on the aesni code. 38 */ 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/malloc.h> 49 #include <sys/mbuf.h> 50 #include <sys/module.h> 51 #include <sys/queue.h> 52 #include <sys/rwlock.h> 53 #include <sys/smp.h> 54 #include <sys/uio.h> 55 56 #include <machine/vfp.h> 57 58 #include <opencrypto/cryptodev.h> 59 #include <cryptodev_if.h> 60 #include <crypto/armv8/armv8_crypto.h> 61 #include <crypto/rijndael/rijndael.h> 62 63 struct armv8_crypto_softc { 64 int dieing; 65 int32_t cid; 66 uint32_t sid; 67 TAILQ_HEAD(armv8_crypto_sessions_head, armv8_crypto_session) sessions; 68 struct rwlock lock; 69 }; 70 71 static struct mtx *ctx_mtx; 72 static struct fpu_kern_ctx **ctx_vfp; 73 74 #define AQUIRE_CTX(i, ctx) \ 75 do { \ 76 (i) = PCPU_GET(cpuid); \ 77 mtx_lock(&ctx_mtx[(i)]); \ 78 (ctx) = ctx_vfp[(i)]; \ 79 } while (0) 80 #define RELEASE_CTX(i, ctx) \ 81 do { \ 82 mtx_unlock(&ctx_mtx[(i)]); \ 83 (i) = -1; \ 84 (ctx) = NULL; \ 85 } while (0) 86 87 static void armv8_crypto_freesession_locked(struct armv8_crypto_softc *, 88 struct armv8_crypto_session *); 89 static int armv8_crypto_cipher_process(struct armv8_crypto_session *, 90 struct cryptodesc *, struct cryptop *); 91 92 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data"); 93 94 static void 95 armv8_crypto_identify(driver_t *drv, device_t parent) 96 { 97 98 /* NB: order 10 is so we get attached after h/w devices */ 99 if (device_find_child(parent, "armv8crypto", -1) == NULL && 100 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0) 101 panic("ARMv8 crypto: could not attach"); 102 } 103 104 static int 105 armv8_crypto_probe(device_t dev) 106 { 107 uint64_t reg; 108 int ret = ENXIO; 109 110 reg = READ_SPECIALREG(id_aa64isar0_el1); 111 112 switch (ID_AA64ISAR0_AES(reg)) { 113 case ID_AA64ISAR0_AES_BASE: 114 case ID_AA64ISAR0_AES_PMULL: 115 ret = 0; 116 break; 117 } 118 119 device_set_desc_copy(dev, "AES-CBC"); 120 121 /* TODO: Check more fields as we support more features */ 122 123 return (ret); 124 } 125 126 static int 127 armv8_crypto_attach(device_t dev) 128 { 129 struct armv8_crypto_softc *sc; 130 int i; 131 132 sc = device_get_softc(dev); 133 TAILQ_INIT(&sc->sessions); 134 sc->dieing = 0; 135 sc->sid = 1; 136 137 sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | 138 CRYPTOCAP_F_SYNC); 139 if (sc->cid < 0) { 140 device_printf(dev, "Could not get crypto driver id.\n"); 141 return (ENOMEM); 142 } 143 144 rw_init(&sc->lock, "armv8crypto"); 145 146 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO, 147 M_WAITOK|M_ZERO); 148 ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO, 149 M_WAITOK|M_ZERO); 150 151 CPU_FOREACH(i) { 152 ctx_vfp[i] = fpu_kern_alloc_ctx(0); 153 mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW); 154 } 155 156 crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); 157 158 return (0); 159 } 160 161 static int 162 armv8_crypto_detach(device_t dev) 163 { 164 struct armv8_crypto_softc *sc; 165 struct armv8_crypto_session *ses; 166 int i; 167 168 sc = device_get_softc(dev); 169 170 rw_wlock(&sc->lock); 171 TAILQ_FOREACH(ses, &sc->sessions, next) { 172 if (ses->used) { 173 rw_wunlock(&sc->lock); 174 device_printf(dev, 175 "Cannot detach, sessions still active.\n"); 176 return (EBUSY); 177 } 178 } 179 sc->dieing = 1; 180 while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) { 181 TAILQ_REMOVE(&sc->sessions, ses, next); 182 free(ses, M_ARMV8_CRYPTO); 183 } 184 rw_wunlock(&sc->lock); 185 crypto_unregister_all(sc->cid); 186 187 rw_destroy(&sc->lock); 188 189 CPU_FOREACH(i) { 190 if (ctx_vfp[i] != NULL) { 191 mtx_destroy(&ctx_mtx[i]); 192 fpu_kern_free_ctx(ctx_vfp[i]); 193 } 194 ctx_vfp[i] = NULL; 195 } 196 free(ctx_mtx, M_ARMV8_CRYPTO); 197 ctx_mtx = NULL; 198 free(ctx_vfp, M_ARMV8_CRYPTO); 199 ctx_vfp = NULL; 200 201 return (0); 202 } 203 204 static int 205 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, 206 struct cryptoini *encini) 207 { 208 int i; 209 210 switch (ses->algo) { 211 case CRYPTO_AES_CBC: 212 switch (encini->cri_klen) { 213 case 128: 214 ses->rounds = AES128_ROUNDS; 215 break; 216 case 192: 217 ses->rounds = AES192_ROUNDS; 218 break; 219 case 256: 220 ses->rounds = AES256_ROUNDS; 221 break; 222 default: 223 CRYPTDEB("invalid CBC/ICM/GCM key length"); 224 return (EINVAL); 225 } 226 break; 227 default: 228 return (EINVAL); 229 } 230 231 rijndaelKeySetupEnc(ses->enc_schedule, encini->cri_key, 232 encini->cri_klen); 233 rijndaelKeySetupDec(ses->dec_schedule, encini->cri_key, 234 encini->cri_klen); 235 for (i = 0; i < nitems(ses->enc_schedule); i++) { 236 ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]); 237 ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]); 238 } 239 240 return (0); 241 } 242 243 static int 244 armv8_crypto_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 245 { 246 struct armv8_crypto_softc *sc; 247 struct armv8_crypto_session *ses; 248 struct cryptoini *encini; 249 int error; 250 251 if (sidp == NULL || cri == NULL) { 252 CRYPTDEB("no sidp or cri"); 253 return (EINVAL); 254 } 255 256 sc = device_get_softc(dev); 257 if (sc->dieing) 258 return (EINVAL); 259 260 ses = NULL; 261 encini = NULL; 262 for (; cri != NULL; cri = cri->cri_next) { 263 switch (cri->cri_alg) { 264 case CRYPTO_AES_CBC: 265 if (encini != NULL) { 266 CRYPTDEB("encini already set"); 267 return (EINVAL); 268 } 269 encini = cri; 270 break; 271 default: 272 CRYPTDEB("unhandled algorithm"); 273 return (EINVAL); 274 } 275 } 276 if (encini == NULL) { 277 CRYPTDEB("no cipher"); 278 return (EINVAL); 279 } 280 281 rw_wlock(&sc->lock); 282 if (sc->dieing) { 283 rw_wunlock(&sc->lock); 284 return (EINVAL); 285 } 286 287 /* 288 * Free sessions goes first, so if first session is used, we need to 289 * allocate one. 290 */ 291 ses = TAILQ_FIRST(&sc->sessions); 292 if (ses == NULL || ses->used) { 293 ses = malloc(sizeof(*ses), M_ARMV8_CRYPTO, M_NOWAIT | M_ZERO); 294 if (ses == NULL) { 295 rw_wunlock(&sc->lock); 296 return (ENOMEM); 297 } 298 ses->id = sc->sid++; 299 } else { 300 TAILQ_REMOVE(&sc->sessions, ses, next); 301 } 302 ses->used = 1; 303 TAILQ_INSERT_TAIL(&sc->sessions, ses, next); 304 rw_wunlock(&sc->lock); 305 ses->algo = encini->cri_alg; 306 307 error = armv8_crypto_cipher_setup(ses, encini); 308 if (error != 0) { 309 CRYPTDEB("setup failed"); 310 rw_wlock(&sc->lock); 311 armv8_crypto_freesession_locked(sc, ses); 312 rw_wunlock(&sc->lock); 313 return (error); 314 } 315 316 *sidp = ses->id; 317 return (0); 318 } 319 320 static void 321 armv8_crypto_freesession_locked(struct armv8_crypto_softc *sc, 322 struct armv8_crypto_session *ses) 323 { 324 uint32_t sid; 325 326 rw_assert(&sc->lock, RA_WLOCKED); 327 328 sid = ses->id; 329 TAILQ_REMOVE(&sc->sessions, ses, next); 330 *ses = (struct armv8_crypto_session){}; 331 ses->id = sid; 332 TAILQ_INSERT_HEAD(&sc->sessions, ses, next); 333 } 334 335 static int 336 armv8_crypto_freesession(device_t dev, uint64_t tid) 337 { 338 struct armv8_crypto_softc *sc; 339 struct armv8_crypto_session *ses; 340 uint32_t sid; 341 342 sc = device_get_softc(dev); 343 sid = ((uint32_t)tid) & 0xffffffff; 344 rw_wlock(&sc->lock); 345 TAILQ_FOREACH_REVERSE(ses, &sc->sessions, armv8_crypto_sessions_head, 346 next) { 347 if (ses->id == sid) 348 break; 349 } 350 if (ses == NULL) { 351 rw_wunlock(&sc->lock); 352 return (EINVAL); 353 } 354 armv8_crypto_freesession_locked(sc, ses); 355 rw_wunlock(&sc->lock); 356 357 return (0); 358 } 359 360 static int 361 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused) 362 { 363 struct armv8_crypto_softc *sc = device_get_softc(dev); 364 struct cryptodesc *crd, *enccrd; 365 struct armv8_crypto_session *ses; 366 int error; 367 368 error = 0; 369 enccrd = NULL; 370 371 /* Sanity check. */ 372 if (crp == NULL) 373 return (EINVAL); 374 375 if (crp->crp_callback == NULL || crp->crp_desc == NULL) { 376 error = EINVAL; 377 goto out; 378 } 379 380 for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { 381 switch (crd->crd_alg) { 382 case CRYPTO_AES_CBC: 383 if (enccrd != NULL) { 384 error = EINVAL; 385 goto out; 386 } 387 enccrd = crd; 388 break; 389 default: 390 error = EINVAL; 391 goto out; 392 } 393 } 394 395 if (enccrd == NULL) { 396 error = EINVAL; 397 goto out; 398 } 399 400 /* We can only handle full blocks for now */ 401 if ((enccrd->crd_len % AES_BLOCK_LEN) != 0) { 402 error = EINVAL; 403 goto out; 404 } 405 406 rw_rlock(&sc->lock); 407 TAILQ_FOREACH_REVERSE(ses, &sc->sessions, armv8_crypto_sessions_head, 408 next) { 409 if (ses->id == (crp->crp_sid & 0xffffffff)) 410 break; 411 } 412 rw_runlock(&sc->lock); 413 if (ses == NULL) { 414 error = EINVAL; 415 goto out; 416 } 417 418 error = armv8_crypto_cipher_process(ses, enccrd, crp); 419 420 out: 421 crp->crp_etype = error; 422 crypto_done(crp); 423 return (error); 424 } 425 426 static uint8_t * 427 armv8_crypto_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, 428 int *allocated) 429 { 430 struct mbuf *m; 431 struct uio *uio; 432 struct iovec *iov; 433 uint8_t *addr; 434 435 if (crp->crp_flags & CRYPTO_F_IMBUF) { 436 m = (struct mbuf *)crp->crp_buf; 437 if (m->m_next != NULL) 438 goto alloc; 439 addr = mtod(m, uint8_t *); 440 } else if (crp->crp_flags & CRYPTO_F_IOV) { 441 uio = (struct uio *)crp->crp_buf; 442 if (uio->uio_iovcnt != 1) 443 goto alloc; 444 iov = uio->uio_iov; 445 addr = (uint8_t *)iov->iov_base; 446 } else 447 addr = (uint8_t *)crp->crp_buf; 448 *allocated = 0; 449 addr += enccrd->crd_skip; 450 return (addr); 451 452 alloc: 453 addr = malloc(enccrd->crd_len, M_ARMV8_CRYPTO, M_NOWAIT); 454 if (addr != NULL) { 455 *allocated = 1; 456 crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 457 enccrd->crd_len, addr); 458 } else 459 *allocated = 0; 460 return (addr); 461 } 462 463 static int 464 armv8_crypto_cipher_process(struct armv8_crypto_session *ses, 465 struct cryptodesc *enccrd, struct cryptop *crp) 466 { 467 struct fpu_kern_ctx *ctx; 468 uint8_t *buf; 469 uint8_t iv[AES_BLOCK_LEN]; 470 int allocated, error, i; 471 int encflag, ivlen; 472 int kt; 473 474 encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT; 475 476 buf = armv8_crypto_cipher_alloc(enccrd, crp, &allocated); 477 if (buf == NULL) 478 return (ENOMEM); 479 480 error = 0; 481 482 kt = is_fpu_kern_thread(0); 483 if (!kt) { 484 AQUIRE_CTX(i, ctx); 485 error = fpu_kern_enter(curthread, ctx, 486 FPU_KERN_NORMAL | FPU_KERN_KTHR); 487 if (error != 0) 488 goto out; 489 } 490 491 if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 492 panic("CRD_F_KEY_EXPLICIT"); 493 } 494 495 switch (enccrd->crd_alg) { 496 case CRYPTO_AES_CBC: 497 ivlen = AES_BLOCK_LEN; 498 break; 499 } 500 501 /* Setup iv */ 502 if (encflag) { 503 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 504 bcopy(enccrd->crd_iv, iv, ivlen); 505 else 506 arc4rand(iv, ivlen, 0); 507 508 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) 509 crypto_copyback(crp->crp_flags, crp->crp_buf, 510 enccrd->crd_inject, ivlen, iv); 511 } else { 512 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 513 bcopy(enccrd->crd_iv, iv, ivlen); 514 else 515 crypto_copydata(crp->crp_flags, crp->crp_buf, 516 enccrd->crd_inject, ivlen, iv); 517 } 518 519 /* Do work */ 520 switch (ses->algo) { 521 case CRYPTO_AES_CBC: 522 if (encflag) 523 armv8_aes_encrypt_cbc(ses->rounds, ses->enc_schedule, 524 enccrd->crd_len, buf, buf, iv); 525 else 526 armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule, 527 enccrd->crd_len, buf, iv); 528 break; 529 } 530 531 if (allocated) 532 crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 533 enccrd->crd_len, buf); 534 535 if (!kt) { 536 fpu_kern_leave(curthread, ctx); 537 out: 538 RELEASE_CTX(i, ctx); 539 } 540 if (allocated) { 541 bzero(buf, enccrd->crd_len); 542 free(buf, M_ARMV8_CRYPTO); 543 } 544 return (error); 545 } 546 547 static device_method_t armv8_crypto_methods[] = { 548 DEVMETHOD(device_identify, armv8_crypto_identify), 549 DEVMETHOD(device_probe, armv8_crypto_probe), 550 DEVMETHOD(device_attach, armv8_crypto_attach), 551 DEVMETHOD(device_detach, armv8_crypto_detach), 552 553 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession), 554 DEVMETHOD(cryptodev_freesession, armv8_crypto_freesession), 555 DEVMETHOD(cryptodev_process, armv8_crypto_process), 556 557 DEVMETHOD_END, 558 }; 559 560 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods, 561 sizeof(struct armv8_crypto_softc)); 562 static devclass_t armv8_crypto_devclass; 563 564 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass, 565 0, 0); 566