1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 19 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 24 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 30 * 3.0 are supported. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/bus.h> 39 #include <sys/endian.h> 40 #include <sys/kernel.h> 41 #include <sys/lock.h> 42 #include <sys/malloc.h> 43 #include <sys/mbuf.h> 44 #include <sys/module.h> 45 #include <sys/mutex.h> 46 #include <sys/random.h> 47 #include <sys/rman.h> 48 49 #include <machine/_inttypes.h> 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform_auth.h> 55 #include "cryptodev_if.h" 56 57 #include <dev/ofw/ofw_bus_subr.h> 58 #include <dev/sec/sec.h> 59 60 static int sec_probe(device_t dev); 61 static int sec_attach(device_t dev); 62 static int sec_detach(device_t dev); 63 static int sec_suspend(device_t dev); 64 static int sec_resume(device_t dev); 65 static int sec_shutdown(device_t dev); 66 static void sec_primary_intr(void *arg); 67 static void sec_secondary_intr(void *arg); 68 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 69 void **ihand, int *irid, driver_intr_t handler, const char *iname); 70 static void sec_release_intr(struct sec_softc *sc, struct resource *ires, 71 void *ihand, int irid, const char *iname); 72 static int sec_controller_reset(struct sec_softc *sc); 73 static int sec_channel_reset(struct sec_softc *sc, int channel, int full); 74 static int sec_init(struct sec_softc *sc); 75 static int sec_alloc_dma_mem(struct sec_softc *sc, 76 struct sec_dma_mem *dma_mem, bus_size_t size); 77 static int sec_desc_map_dma(struct sec_softc *sc, 78 struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size, 79 struct sec_desc_map_info *sdmi); 80 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 81 static void sec_enqueue(struct sec_softc *sc); 82 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 83 int channel); 84 static int sec_eu_channel(struct sec_softc *sc, int eu); 85 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 86 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize); 87 static int sec_make_pointer_direct(struct sec_softc *sc, 88 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 89 static int sec_probesession(device_t dev, 90 const struct crypto_session_params *csp); 91 static int sec_newsession(device_t dev, crypto_session_t cses, 92 const struct crypto_session_params *csp); 93 static int sec_process(device_t dev, struct cryptop *crp, int hint); 94 static int sec_build_common_ns_desc(struct sec_softc *sc, 95 struct sec_desc *desc, const struct crypto_session_params *csp, 96 struct cryptop *crp); 97 static int sec_build_common_s_desc(struct sec_softc *sc, 98 struct sec_desc *desc, const struct crypto_session_params *csp, 99 struct cryptop *crp); 100 101 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 102 103 /* AESU */ 104 static bool sec_aesu_newsession(const struct crypto_session_params *csp); 105 static int sec_aesu_make_desc(struct sec_softc *sc, 106 const struct crypto_session_params *csp, struct sec_desc *desc, 107 struct cryptop *crp); 108 109 /* MDEU */ 110 static bool sec_mdeu_can_handle(u_int alg); 111 static int sec_mdeu_config(const struct crypto_session_params *csp, 112 u_int *eu, u_int *mode, u_int *hashlen); 113 static bool sec_mdeu_newsession(const struct crypto_session_params *csp); 114 static int sec_mdeu_make_desc(struct sec_softc *sc, 115 const struct crypto_session_params *csp, struct sec_desc *desc, 116 struct cryptop *crp); 117 118 static device_method_t sec_methods[] = { 119 /* Device interface */ 120 DEVMETHOD(device_probe, sec_probe), 121 DEVMETHOD(device_attach, sec_attach), 122 DEVMETHOD(device_detach, sec_detach), 123 124 DEVMETHOD(device_suspend, sec_suspend), 125 DEVMETHOD(device_resume, sec_resume), 126 DEVMETHOD(device_shutdown, sec_shutdown), 127 128 /* Crypto methods */ 129 DEVMETHOD(cryptodev_probesession, sec_probesession), 130 DEVMETHOD(cryptodev_newsession, sec_newsession), 131 DEVMETHOD(cryptodev_process, sec_process), 132 133 DEVMETHOD_END 134 }; 135 static driver_t sec_driver = { 136 "sec", 137 sec_methods, 138 sizeof(struct sec_softc), 139 }; 140 141 DRIVER_MODULE(sec, simplebus, sec_driver, 0, 0); 142 MODULE_DEPEND(sec, crypto, 1, 1, 1); 143 144 static struct sec_eu_methods sec_eus[] = { 145 { 146 sec_aesu_newsession, 147 sec_aesu_make_desc, 148 }, 149 { 150 sec_mdeu_newsession, 151 sec_mdeu_make_desc, 152 }, 153 { NULL, NULL } 154 }; 155 156 static inline void 157 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 158 { 159 160 /* Sync only if dma memory is valid */ 161 if (dma_mem->dma_vaddr != NULL) 162 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 163 } 164 165 static inline void * 166 sec_get_pointer_data(struct sec_desc *desc, u_int n) 167 { 168 169 return (desc->sd_ptr_dmem[n].dma_vaddr); 170 } 171 172 static int 173 sec_probe(device_t dev) 174 { 175 struct sec_softc *sc; 176 uint64_t id; 177 178 if (!ofw_bus_status_okay(dev)) 179 return (ENXIO); 180 181 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0")) 182 return (ENXIO); 183 184 sc = device_get_softc(dev); 185 186 sc->sc_rrid = 0; 187 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 188 RF_ACTIVE); 189 190 if (sc->sc_rres == NULL) 191 return (ENXIO); 192 193 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 194 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 195 196 id = SEC_READ(sc, SEC_ID); 197 198 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 199 200 switch (id) { 201 case SEC_20_ID: 202 device_set_desc(dev, "Freescale Security Engine 2.0"); 203 sc->sc_version = 2; 204 break; 205 case SEC_30_ID: 206 device_set_desc(dev, "Freescale Security Engine 3.0"); 207 sc->sc_version = 3; 208 break; 209 case SEC_31_ID: 210 device_set_desc(dev, "Freescale Security Engine 3.1"); 211 sc->sc_version = 3; 212 break; 213 default: 214 device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id); 215 return (ENXIO); 216 } 217 218 return (0); 219 } 220 221 static int 222 sec_attach(device_t dev) 223 { 224 struct sec_softc *sc; 225 struct sec_hw_lt *lt; 226 int error = 0; 227 int i; 228 229 sc = device_get_softc(dev); 230 sc->sc_dev = dev; 231 sc->sc_blocked = 0; 232 sc->sc_shutdown = 0; 233 234 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session), 235 CRYPTOCAP_F_HARDWARE); 236 if (sc->sc_cid < 0) { 237 device_printf(dev, "could not get crypto driver ID!\n"); 238 return (ENXIO); 239 } 240 241 /* Init locks */ 242 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 243 "SEC Controller lock", MTX_DEF); 244 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 245 "SEC Descriptors lock", MTX_DEF); 246 247 /* Allocate I/O memory for SEC registers */ 248 sc->sc_rrid = 0; 249 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 250 RF_ACTIVE); 251 252 if (sc->sc_rres == NULL) { 253 device_printf(dev, "could not allocate I/O memory!\n"); 254 goto fail1; 255 } 256 257 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 258 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 259 260 /* Setup interrupts */ 261 sc->sc_pri_irid = 0; 262 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 263 &sc->sc_pri_irid, sec_primary_intr, "primary"); 264 265 if (error) 266 goto fail2; 267 268 if (sc->sc_version == 3) { 269 sc->sc_sec_irid = 1; 270 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 271 &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 272 273 if (error) 274 goto fail3; 275 } 276 277 /* Alloc DMA memory for descriptors and link tables */ 278 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 279 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 280 281 if (error) 282 goto fail4; 283 284 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 285 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 286 287 if (error) 288 goto fail5; 289 290 /* Fill in descriptors and link tables */ 291 for (i = 0; i < SEC_DESCRIPTORS; i++) { 292 sc->sc_desc[i].sd_desc = 293 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 294 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 295 (i * sizeof(struct sec_hw_desc)); 296 } 297 298 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 299 sc->sc_lt[i].sl_lt = 300 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 301 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 302 (i * sizeof(struct sec_hw_lt)); 303 } 304 305 /* Last entry in link table is used to create a circle */ 306 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 307 lt->shl_length = 0; 308 lt->shl_r = 0; 309 lt->shl_n = 1; 310 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 311 312 /* Init descriptor and link table queues pointers */ 313 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 314 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 315 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 316 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 317 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 318 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 319 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 320 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 321 322 /* Create masks for fast checks */ 323 sc->sc_int_error_mask = 0; 324 for (i = 0; i < SEC_CHANNELS; i++) 325 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 326 327 switch (sc->sc_version) { 328 case 2: 329 sc->sc_channel_idle_mask = 330 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 331 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 332 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 333 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 334 break; 335 case 3: 336 sc->sc_channel_idle_mask = 337 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 338 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 339 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 340 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 341 break; 342 } 343 344 /* Init hardware */ 345 error = sec_init(sc); 346 347 if (error) 348 goto fail6; 349 350 return (0); 351 352 fail6: 353 sec_free_dma_mem(&(sc->sc_lt_dmem)); 354 fail5: 355 sec_free_dma_mem(&(sc->sc_desc_dmem)); 356 fail4: 357 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 358 sc->sc_sec_irid, "secondary"); 359 fail3: 360 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 361 sc->sc_pri_irid, "primary"); 362 fail2: 363 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 364 fail1: 365 mtx_destroy(&sc->sc_controller_lock); 366 mtx_destroy(&sc->sc_descriptors_lock); 367 368 return (ENXIO); 369 } 370 371 static int 372 sec_detach(device_t dev) 373 { 374 struct sec_softc *sc = device_get_softc(dev); 375 int i, error, timeout = SEC_TIMEOUT; 376 377 /* Prepare driver to shutdown */ 378 SEC_LOCK(sc, descriptors); 379 sc->sc_shutdown = 1; 380 SEC_UNLOCK(sc, descriptors); 381 382 /* Wait until all queued processing finishes */ 383 while (1) { 384 SEC_LOCK(sc, descriptors); 385 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 386 SEC_UNLOCK(sc, descriptors); 387 388 if (i == 0) 389 break; 390 391 if (timeout < 0) { 392 device_printf(dev, "queue flush timeout!\n"); 393 394 /* DMA can be still active - stop it */ 395 for (i = 0; i < SEC_CHANNELS; i++) 396 sec_channel_reset(sc, i, 1); 397 398 break; 399 } 400 401 timeout -= 1000; 402 DELAY(1000); 403 } 404 405 /* Disable interrupts */ 406 SEC_WRITE(sc, SEC_IER, 0); 407 408 /* Unregister from OCF */ 409 crypto_unregister_all(sc->sc_cid); 410 411 /* Free DMA memory */ 412 for (i = 0; i < SEC_DESCRIPTORS; i++) 413 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 414 415 sec_free_dma_mem(&(sc->sc_lt_dmem)); 416 sec_free_dma_mem(&(sc->sc_desc_dmem)); 417 418 /* Release interrupts */ 419 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 420 sc->sc_pri_irid, "primary"); 421 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 422 sc->sc_sec_irid, "secondary"); 423 424 /* Release memory */ 425 if (sc->sc_rres) { 426 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 427 sc->sc_rres); 428 if (error) 429 device_printf(dev, "bus_release_resource() failed for" 430 " I/O memory, error %d\n", error); 431 432 sc->sc_rres = NULL; 433 } 434 435 mtx_destroy(&sc->sc_controller_lock); 436 mtx_destroy(&sc->sc_descriptors_lock); 437 438 return (0); 439 } 440 441 static int 442 sec_suspend(device_t dev) 443 { 444 445 return (0); 446 } 447 448 static int 449 sec_resume(device_t dev) 450 { 451 452 return (0); 453 } 454 455 static int 456 sec_shutdown(device_t dev) 457 { 458 459 return (0); 460 } 461 462 static int 463 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 464 int *irid, driver_intr_t handler, const char *iname) 465 { 466 int error; 467 468 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 469 RF_ACTIVE); 470 471 if ((*ires) == NULL) { 472 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 473 return (ENXIO); 474 } 475 476 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 477 NULL, handler, sc, ihand); 478 479 if (error) { 480 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 481 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 482 device_printf(sc->sc_dev, "could not release %s IRQ\n", 483 iname); 484 485 (*ires) = NULL; 486 return (error); 487 } 488 489 return (0); 490 } 491 492 static void 493 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 494 int irid, const char *iname) 495 { 496 int error; 497 498 if (ires == NULL) 499 return; 500 501 error = bus_teardown_intr(sc->sc_dev, ires, ihand); 502 if (error) 503 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 504 " IRQ, error %d\n", iname, error); 505 506 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 507 if (error) 508 device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 509 " IRQ, error %d\n", iname, error); 510 } 511 512 static void 513 sec_primary_intr(void *arg) 514 { 515 struct sec_session *ses; 516 struct sec_softc *sc = arg; 517 struct sec_desc *desc; 518 struct cryptop *crp; 519 uint64_t isr; 520 uint8_t hash[HASH_MAX_LEN]; 521 int i, wakeup = 0; 522 523 SEC_LOCK(sc, controller); 524 525 /* Check for errors */ 526 isr = SEC_READ(sc, SEC_ISR); 527 if (isr & sc->sc_int_error_mask) { 528 /* Check each channel for error */ 529 for (i = 0; i < SEC_CHANNELS; i++) { 530 if ((isr & SEC_INT_CH_ERR(i)) == 0) 531 continue; 532 533 device_printf(sc->sc_dev, 534 "I/O error on channel %i!\n", i); 535 536 /* Find and mark problematic descriptor */ 537 desc = sec_find_desc(sc, SEC_READ(sc, 538 SEC_CHAN_CDPR(i))); 539 540 if (desc != NULL) 541 desc->sd_error = EIO; 542 543 /* Do partial channel reset */ 544 sec_channel_reset(sc, i, 0); 545 } 546 } 547 548 /* ACK interrupt */ 549 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 550 551 SEC_UNLOCK(sc, controller); 552 SEC_LOCK(sc, descriptors); 553 554 /* Handle processed descriptors */ 555 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 556 557 while (SEC_QUEUED_DESC_CNT(sc) > 0) { 558 desc = SEC_GET_QUEUED_DESC(sc); 559 560 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 561 SEC_PUT_BACK_QUEUED_DESC(sc); 562 break; 563 } 564 565 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 566 BUS_DMASYNC_PREWRITE); 567 568 crp = desc->sd_crp; 569 crp->crp_etype = desc->sd_error; 570 if (crp->crp_etype == 0) { 571 ses = crypto_get_driver_session(crp->crp_session); 572 if (ses->ss_mlen != 0) { 573 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 574 crypto_copydata(crp, 575 crp->crp_digest_start, 576 ses->ss_mlen, hash); 577 if (timingsafe_bcmp( 578 desc->sd_desc->shd_digest, 579 hash, ses->ss_mlen) != 0) 580 crp->crp_etype = EBADMSG; 581 } else 582 crypto_copyback(crp, 583 crp->crp_digest_start, 584 ses->ss_mlen, 585 desc->sd_desc->shd_digest); 586 } 587 } 588 crypto_done(desc->sd_crp); 589 590 SEC_DESC_FREE_POINTERS(desc); 591 SEC_DESC_FREE_LT(sc, desc); 592 SEC_DESC_QUEUED2FREE(sc); 593 } 594 595 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 596 597 if (!sc->sc_shutdown) { 598 wakeup = sc->sc_blocked; 599 sc->sc_blocked = 0; 600 } 601 602 SEC_UNLOCK(sc, descriptors); 603 604 /* Enqueue ready descriptors in hardware */ 605 sec_enqueue(sc); 606 607 if (wakeup) 608 crypto_unblock(sc->sc_cid, wakeup); 609 } 610 611 static void 612 sec_secondary_intr(void *arg) 613 { 614 struct sec_softc *sc = arg; 615 616 device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 617 sec_primary_intr(arg); 618 } 619 620 static int 621 sec_controller_reset(struct sec_softc *sc) 622 { 623 int timeout = SEC_TIMEOUT; 624 625 /* Reset Controller */ 626 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 627 628 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 629 DELAY(1000); 630 timeout -= 1000; 631 632 if (timeout < 0) { 633 device_printf(sc->sc_dev, "timeout while waiting for " 634 "device reset!\n"); 635 return (ETIMEDOUT); 636 } 637 } 638 639 return (0); 640 } 641 642 static int 643 sec_channel_reset(struct sec_softc *sc, int channel, int full) 644 { 645 int timeout = SEC_TIMEOUT; 646 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 647 uint64_t reg; 648 649 /* Reset Channel */ 650 reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 651 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 652 653 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 654 DELAY(1000); 655 timeout -= 1000; 656 657 if (timeout < 0) { 658 device_printf(sc->sc_dev, "timeout while waiting for " 659 "channel reset!\n"); 660 return (ETIMEDOUT); 661 } 662 } 663 664 if (full) { 665 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 666 667 switch(sc->sc_version) { 668 case 2: 669 reg |= SEC_CHAN_CCR_CDWE; 670 break; 671 case 3: 672 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 673 break; 674 } 675 676 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 677 } 678 679 return (0); 680 } 681 682 static int 683 sec_init(struct sec_softc *sc) 684 { 685 uint64_t reg; 686 int error, i; 687 688 /* Reset controller twice to clear all pending interrupts */ 689 error = sec_controller_reset(sc); 690 if (error) 691 return (error); 692 693 error = sec_controller_reset(sc); 694 if (error) 695 return (error); 696 697 /* Reset channels */ 698 for (i = 0; i < SEC_CHANNELS; i++) { 699 error = sec_channel_reset(sc, i, 1); 700 if (error) 701 return (error); 702 } 703 704 /* Enable Interrupts */ 705 reg = SEC_INT_ITO; 706 for (i = 0; i < SEC_CHANNELS; i++) 707 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 708 709 SEC_WRITE(sc, SEC_IER, reg); 710 711 return (error); 712 } 713 714 static void 715 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 716 { 717 struct sec_dma_mem *dma_mem = arg; 718 719 if (error) 720 return; 721 722 KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 723 dma_mem->dma_paddr = segs->ds_addr; 724 } 725 726 static void 727 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 728 int error) 729 { 730 struct sec_desc_map_info *sdmi = arg; 731 struct sec_softc *sc = sdmi->sdmi_sc; 732 struct sec_lt *lt = NULL; 733 bus_addr_t addr; 734 bus_size_t size; 735 int i; 736 737 SEC_LOCK_ASSERT(sc, descriptors); 738 739 if (error) 740 return; 741 742 for (i = 0; i < nseg; i++) { 743 addr = segs[i].ds_addr; 744 size = segs[i].ds_len; 745 746 /* Skip requested offset */ 747 if (sdmi->sdmi_offset >= size) { 748 sdmi->sdmi_offset -= size; 749 continue; 750 } 751 752 addr += sdmi->sdmi_offset; 753 size -= sdmi->sdmi_offset; 754 sdmi->sdmi_offset = 0; 755 756 /* Do not link more than requested */ 757 if (sdmi->sdmi_size < size) 758 size = sdmi->sdmi_size; 759 760 lt = SEC_ALLOC_LT_ENTRY(sc); 761 lt->sl_lt->shl_length = size; 762 lt->sl_lt->shl_r = 0; 763 lt->sl_lt->shl_n = 0; 764 lt->sl_lt->shl_ptr = addr; 765 766 if (sdmi->sdmi_lt_first == NULL) 767 sdmi->sdmi_lt_first = lt; 768 769 sdmi->sdmi_lt_used += 1; 770 771 if ((sdmi->sdmi_size -= size) == 0) 772 break; 773 } 774 775 sdmi->sdmi_lt_last = lt; 776 } 777 778 static int 779 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 780 bus_size_t size) 781 { 782 int error; 783 784 if (dma_mem->dma_vaddr != NULL) 785 return (EBUSY); 786 787 error = bus_dma_tag_create(NULL, /* parent */ 788 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 789 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 790 BUS_SPACE_MAXADDR, /* highaddr */ 791 NULL, NULL, /* filtfunc, filtfuncarg */ 792 size, 1, /* maxsize, nsegments */ 793 size, 0, /* maxsegsz, flags */ 794 NULL, NULL, /* lockfunc, lockfuncarg */ 795 &(dma_mem->dma_tag)); /* dmat */ 796 797 if (error) { 798 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 799 " %i!\n", error); 800 goto err1; 801 } 802 803 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 804 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 805 806 if (error) { 807 device_printf(sc->sc_dev, "failed to allocate DMA safe" 808 " memory, error %i!\n", error); 809 goto err2; 810 } 811 812 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 813 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 814 BUS_DMA_NOWAIT); 815 816 if (error) { 817 device_printf(sc->sc_dev, "cannot get address of the DMA" 818 " memory, error %i\n", error); 819 goto err3; 820 } 821 822 dma_mem->dma_is_map = 0; 823 return (0); 824 825 err3: 826 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 827 err2: 828 bus_dma_tag_destroy(dma_mem->dma_tag); 829 err1: 830 dma_mem->dma_vaddr = NULL; 831 return(error); 832 } 833 834 static int 835 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 836 struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi) 837 { 838 int error; 839 840 if (dma_mem->dma_vaddr != NULL) 841 return (EBUSY); 842 843 switch (crp->crp_buf.cb_type) { 844 case CRYPTO_BUF_CONTIG: 845 break; 846 case CRYPTO_BUF_UIO: 847 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 848 break; 849 case CRYPTO_BUF_MBUF: 850 size = m_length(crp->crp_buf.cb_mbuf, NULL); 851 break; 852 case CRYPTO_BUF_SINGLE_MBUF: 853 size = crp->crp_buf.cb_mbuf->m_len; 854 break; 855 case CRYPTO_BUF_VMPAGE: 856 size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset; 857 break; 858 default: 859 return (EINVAL); 860 } 861 862 error = bus_dma_tag_create(NULL, /* parent */ 863 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 864 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 865 BUS_SPACE_MAXADDR, /* highaddr */ 866 NULL, NULL, /* filtfunc, filtfuncarg */ 867 size, /* maxsize */ 868 SEC_FREE_LT_CNT(sc), /* nsegments */ 869 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 870 NULL, NULL, /* lockfunc, lockfuncarg */ 871 &(dma_mem->dma_tag)); /* dmat */ 872 873 if (error) { 874 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 875 " %i!\n", error); 876 dma_mem->dma_vaddr = NULL; 877 return (error); 878 } 879 880 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 881 882 if (error) { 883 device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 884 "\n", error); 885 bus_dma_tag_destroy(dma_mem->dma_tag); 886 return (error); 887 } 888 889 error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp, 890 sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 891 892 if (error) { 893 device_printf(sc->sc_dev, "cannot get address of the DMA" 894 " memory, error %i!\n", error); 895 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 896 bus_dma_tag_destroy(dma_mem->dma_tag); 897 return (error); 898 } 899 900 dma_mem->dma_is_map = 1; 901 dma_mem->dma_vaddr = crp; 902 903 return (0); 904 } 905 906 static void 907 sec_free_dma_mem(struct sec_dma_mem *dma_mem) 908 { 909 910 /* Check for double free */ 911 if (dma_mem->dma_vaddr == NULL) 912 return; 913 914 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 915 916 if (dma_mem->dma_is_map) 917 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 918 else 919 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 920 dma_mem->dma_map); 921 922 bus_dma_tag_destroy(dma_mem->dma_tag); 923 dma_mem->dma_vaddr = NULL; 924 } 925 926 static int 927 sec_eu_channel(struct sec_softc *sc, int eu) 928 { 929 uint64_t reg; 930 int channel = 0; 931 932 SEC_LOCK_ASSERT(sc, controller); 933 934 reg = SEC_READ(sc, SEC_EUASR); 935 936 switch (eu) { 937 case SEC_EU_AFEU: 938 channel = SEC_EUASR_AFEU(reg); 939 break; 940 case SEC_EU_DEU: 941 channel = SEC_EUASR_DEU(reg); 942 break; 943 case SEC_EU_MDEU_A: 944 case SEC_EU_MDEU_B: 945 channel = SEC_EUASR_MDEU(reg); 946 break; 947 case SEC_EU_RNGU: 948 channel = SEC_EUASR_RNGU(reg); 949 break; 950 case SEC_EU_PKEU: 951 channel = SEC_EUASR_PKEU(reg); 952 break; 953 case SEC_EU_AESU: 954 channel = SEC_EUASR_AESU(reg); 955 break; 956 case SEC_EU_KEU: 957 channel = SEC_EUASR_KEU(reg); 958 break; 959 case SEC_EU_CRCU: 960 channel = SEC_EUASR_CRCU(reg); 961 break; 962 } 963 964 return (channel - 1); 965 } 966 967 static int 968 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 969 { 970 u_int fflvl = SEC_MAX_FIFO_LEVEL; 971 uint64_t reg; 972 int i; 973 974 SEC_LOCK_ASSERT(sc, controller); 975 976 /* Find free channel if have not got one */ 977 if (channel < 0) { 978 for (i = 0; i < SEC_CHANNELS; i++) { 979 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 980 981 if ((reg & sc->sc_channel_idle_mask) == 0) { 982 channel = i; 983 break; 984 } 985 } 986 } 987 988 /* There is no free channel */ 989 if (channel < 0) 990 return (-1); 991 992 /* Check FIFO level on selected channel */ 993 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 994 995 switch(sc->sc_version) { 996 case 2: 997 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 998 break; 999 case 3: 1000 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 1001 break; 1002 } 1003 1004 if (fflvl >= SEC_MAX_FIFO_LEVEL) 1005 return (-1); 1006 1007 /* Enqueue descriptor in channel */ 1008 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1009 1010 return (channel); 1011 } 1012 1013 static void 1014 sec_enqueue(struct sec_softc *sc) 1015 { 1016 struct sec_desc *desc; 1017 int ch0, ch1; 1018 1019 SEC_LOCK(sc, descriptors); 1020 SEC_LOCK(sc, controller); 1021 1022 while (SEC_READY_DESC_CNT(sc) > 0) { 1023 desc = SEC_GET_READY_DESC(sc); 1024 1025 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1026 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1027 1028 /* 1029 * Both EU are used by the same channel. 1030 * Enqueue descriptor in channel used by busy EUs. 1031 */ 1032 if (ch0 >= 0 && ch0 == ch1) { 1033 if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1034 SEC_DESC_READY2QUEUED(sc); 1035 continue; 1036 } 1037 } 1038 1039 /* 1040 * Only one EU is free. 1041 * Enqueue descriptor in channel used by busy EU. 1042 */ 1043 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1044 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1045 >= 0) { 1046 SEC_DESC_READY2QUEUED(sc); 1047 continue; 1048 } 1049 } 1050 1051 /* 1052 * Both EU are free. 1053 * Enqueue descriptor in first free channel. 1054 */ 1055 if (ch0 < 0 && ch1 < 0) { 1056 if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1057 SEC_DESC_READY2QUEUED(sc); 1058 continue; 1059 } 1060 } 1061 1062 /* Current descriptor can not be queued at the moment */ 1063 SEC_PUT_BACK_READY_DESC(sc); 1064 break; 1065 } 1066 1067 SEC_UNLOCK(sc, controller); 1068 SEC_UNLOCK(sc, descriptors); 1069 } 1070 1071 static struct sec_desc * 1072 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1073 { 1074 struct sec_desc *desc = NULL; 1075 int i; 1076 1077 SEC_LOCK_ASSERT(sc, descriptors); 1078 1079 for (i = 0; i < SEC_CHANNELS; i++) { 1080 if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1081 desc = &(sc->sc_desc[i]); 1082 break; 1083 } 1084 } 1085 1086 return (desc); 1087 } 1088 1089 static int 1090 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1091 bus_addr_t data, bus_size_t dsize) 1092 { 1093 struct sec_hw_desc_ptr *ptr; 1094 1095 SEC_LOCK_ASSERT(sc, descriptors); 1096 1097 ptr = &(desc->sd_desc->shd_pointer[n]); 1098 ptr->shdp_length = dsize; 1099 ptr->shdp_extent = 0; 1100 ptr->shdp_j = 0; 1101 ptr->shdp_ptr = data; 1102 1103 return (0); 1104 } 1105 1106 static int 1107 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1108 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize) 1109 { 1110 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1111 struct sec_hw_desc_ptr *ptr; 1112 int error; 1113 1114 SEC_LOCK_ASSERT(sc, descriptors); 1115 1116 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize, 1117 &sdmi); 1118 1119 if (error) 1120 return (error); 1121 1122 sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1123 desc->sd_lt_used += sdmi.sdmi_lt_used; 1124 1125 ptr = &(desc->sd_desc->shd_pointer[n]); 1126 ptr->shdp_length = dsize; 1127 ptr->shdp_extent = 0; 1128 ptr->shdp_j = 1; 1129 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1130 1131 return (0); 1132 } 1133 1134 static bool 1135 sec_cipher_supported(const struct crypto_session_params *csp) 1136 { 1137 1138 switch (csp->csp_cipher_alg) { 1139 case CRYPTO_AES_CBC: 1140 /* AESU */ 1141 if (csp->csp_ivlen != AES_BLOCK_LEN) 1142 return (false); 1143 break; 1144 default: 1145 return (false); 1146 } 1147 1148 if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN) 1149 return (false); 1150 1151 return (true); 1152 } 1153 1154 static bool 1155 sec_auth_supported(struct sec_softc *sc, 1156 const struct crypto_session_params *csp) 1157 { 1158 1159 switch (csp->csp_auth_alg) { 1160 case CRYPTO_SHA2_384_HMAC: 1161 case CRYPTO_SHA2_512_HMAC: 1162 if (sc->sc_version < 3) 1163 return (false); 1164 /* FALLTHROUGH */ 1165 case CRYPTO_SHA1_HMAC: 1166 case CRYPTO_SHA2_256_HMAC: 1167 if (csp->csp_auth_klen > SEC_MAX_KEY_LEN) 1168 return (false); 1169 break; 1170 case CRYPTO_SHA1: 1171 break; 1172 default: 1173 return (false); 1174 } 1175 return (true); 1176 } 1177 1178 static int 1179 sec_probesession(device_t dev, const struct crypto_session_params *csp) 1180 { 1181 struct sec_softc *sc = device_get_softc(dev); 1182 1183 if (csp->csp_flags != 0) 1184 return (EINVAL); 1185 switch (csp->csp_mode) { 1186 case CSP_MODE_DIGEST: 1187 if (!sec_auth_supported(sc, csp)) 1188 return (EINVAL); 1189 break; 1190 case CSP_MODE_CIPHER: 1191 if (!sec_cipher_supported(csp)) 1192 return (EINVAL); 1193 break; 1194 case CSP_MODE_ETA: 1195 if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp)) 1196 return (EINVAL); 1197 break; 1198 default: 1199 return (EINVAL); 1200 } 1201 return (CRYPTODEV_PROBE_HARDWARE); 1202 } 1203 1204 static int 1205 sec_newsession(device_t dev, crypto_session_t cses, 1206 const struct crypto_session_params *csp) 1207 { 1208 struct sec_eu_methods *eu = sec_eus; 1209 struct sec_session *ses; 1210 1211 ses = crypto_get_driver_session(cses); 1212 1213 /* Find EU for this session */ 1214 while (eu->sem_make_desc != NULL) { 1215 if (eu->sem_newsession(csp)) 1216 break; 1217 eu++; 1218 } 1219 KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session")); 1220 1221 /* Save cipher key */ 1222 if (csp->csp_cipher_key != NULL) 1223 memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen); 1224 1225 /* Save digest key */ 1226 if (csp->csp_auth_key != NULL) 1227 memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen); 1228 1229 if (csp->csp_auth_alg != 0) { 1230 if (csp->csp_auth_mlen == 0) 1231 ses->ss_mlen = crypto_auth_hash(csp)->hashsize; 1232 else 1233 ses->ss_mlen = csp->csp_auth_mlen; 1234 } 1235 1236 return (0); 1237 } 1238 1239 static int 1240 sec_process(device_t dev, struct cryptop *crp, int hint) 1241 { 1242 struct sec_softc *sc = device_get_softc(dev); 1243 struct sec_desc *desc = NULL; 1244 const struct crypto_session_params *csp; 1245 struct sec_session *ses; 1246 int error = 0; 1247 1248 ses = crypto_get_driver_session(crp->crp_session); 1249 csp = crypto_get_params(crp->crp_session); 1250 1251 /* Check for input length */ 1252 if (crypto_buffer_len(&crp->crp_buf) > SEC_MAX_DMA_BLOCK_SIZE) { 1253 crp->crp_etype = E2BIG; 1254 crypto_done(crp); 1255 return (0); 1256 } 1257 1258 SEC_LOCK(sc, descriptors); 1259 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1260 1261 /* Block driver if there is no free descriptors or we are going down */ 1262 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1263 sc->sc_blocked |= CRYPTO_SYMQ; 1264 SEC_UNLOCK(sc, descriptors); 1265 return (ERESTART); 1266 } 1267 1268 /* Prepare descriptor */ 1269 desc = SEC_GET_FREE_DESC(sc); 1270 desc->sd_lt_used = 0; 1271 desc->sd_error = 0; 1272 desc->sd_crp = crp; 1273 1274 if (csp->csp_cipher_alg != 0) 1275 crypto_read_iv(crp, desc->sd_desc->shd_iv); 1276 1277 if (crp->crp_cipher_key != NULL) 1278 memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen); 1279 1280 if (crp->crp_auth_key != NULL) 1281 memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen); 1282 1283 memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen); 1284 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen); 1285 1286 error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp); 1287 1288 if (error) { 1289 SEC_DESC_FREE_POINTERS(desc); 1290 SEC_DESC_PUT_BACK_LT(sc, desc); 1291 SEC_PUT_BACK_FREE_DESC(sc); 1292 SEC_UNLOCK(sc, descriptors); 1293 crp->crp_etype = error; 1294 crypto_done(crp); 1295 return (0); 1296 } 1297 1298 /* 1299 * Skip DONE interrupt if this is not last request in burst, but only 1300 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1301 * signaling on each descriptor. 1302 */ 1303 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1304 desc->sd_desc->shd_dn = 0; 1305 else 1306 desc->sd_desc->shd_dn = 1; 1307 1308 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1309 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1310 BUS_DMASYNC_POSTWRITE); 1311 SEC_DESC_FREE2READY(sc); 1312 SEC_UNLOCK(sc, descriptors); 1313 1314 /* Enqueue ready descriptors in hardware */ 1315 sec_enqueue(sc); 1316 1317 return (0); 1318 } 1319 1320 static int 1321 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1322 const struct crypto_session_params *csp, struct cryptop *crp) 1323 { 1324 struct sec_hw_desc *hd = desc->sd_desc; 1325 int error; 1326 1327 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1328 hd->shd_eu_sel1 = SEC_EU_NONE; 1329 hd->shd_mode1 = 0; 1330 1331 /* Pointer 0: NULL */ 1332 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1333 if (error) 1334 return (error); 1335 1336 /* Pointer 1: IV IN */ 1337 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1338 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); 1339 if (error) 1340 return (error); 1341 1342 /* Pointer 2: Cipher Key */ 1343 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1344 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); 1345 if (error) 1346 return (error); 1347 1348 /* Pointer 3: Data IN */ 1349 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, 1350 crp->crp_payload_length); 1351 if (error) 1352 return (error); 1353 1354 /* Pointer 4: Data OUT */ 1355 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, 1356 crp->crp_payload_length); 1357 if (error) 1358 return (error); 1359 1360 /* Pointer 5: IV OUT (Not used: NULL) */ 1361 error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1362 if (error) 1363 return (error); 1364 1365 /* Pointer 6: NULL */ 1366 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1367 1368 return (error); 1369 } 1370 1371 static int 1372 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1373 const struct crypto_session_params *csp, struct cryptop *crp) 1374 { 1375 struct sec_hw_desc *hd = desc->sd_desc; 1376 u_int eu, mode, hashlen; 1377 int error; 1378 1379 error = sec_mdeu_config(csp, &eu, &mode, &hashlen); 1380 if (error) 1381 return (error); 1382 1383 hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1384 hd->shd_eu_sel1 = eu; 1385 hd->shd_mode1 = mode; 1386 1387 /* Pointer 0: HMAC Key */ 1388 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1389 offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen); 1390 if (error) 1391 return (error); 1392 1393 /* Pointer 1: HMAC-Only Data IN */ 1394 error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start, 1395 crp->crp_aad_length); 1396 if (error) 1397 return (error); 1398 1399 /* Pointer 2: Cipher Key */ 1400 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1401 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); 1402 if (error) 1403 return (error); 1404 1405 /* Pointer 3: IV IN */ 1406 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1407 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); 1408 if (error) 1409 return (error); 1410 1411 /* Pointer 4: Data IN */ 1412 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, 1413 crp->crp_payload_length); 1414 if (error) 1415 return (error); 1416 1417 /* Pointer 5: Data OUT */ 1418 error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start, 1419 crp->crp_payload_length); 1420 if (error) 1421 return (error); 1422 1423 /* Pointer 6: HMAC OUT */ 1424 error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr + 1425 offsetof(struct sec_hw_desc, shd_digest), hashlen); 1426 1427 return (error); 1428 } 1429 1430 /* AESU */ 1431 1432 static bool 1433 sec_aesu_newsession(const struct crypto_session_params *csp) 1434 { 1435 1436 return (csp->csp_cipher_alg == CRYPTO_AES_CBC); 1437 } 1438 1439 static int 1440 sec_aesu_make_desc(struct sec_softc *sc, 1441 const struct crypto_session_params *csp, struct sec_desc *desc, 1442 struct cryptop *crp) 1443 { 1444 struct sec_hw_desc *hd = desc->sd_desc; 1445 int error; 1446 1447 hd->shd_eu_sel0 = SEC_EU_AESU; 1448 hd->shd_mode0 = SEC_AESU_MODE_CBC; 1449 1450 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1451 hd->shd_mode0 |= SEC_AESU_MODE_ED; 1452 hd->shd_dir = 0; 1453 } else 1454 hd->shd_dir = 1; 1455 1456 if (csp->csp_mode == CSP_MODE_ETA) 1457 error = sec_build_common_s_desc(sc, desc, csp, crp); 1458 else 1459 error = sec_build_common_ns_desc(sc, desc, csp, crp); 1460 1461 return (error); 1462 } 1463 1464 /* MDEU */ 1465 1466 static bool 1467 sec_mdeu_can_handle(u_int alg) 1468 { 1469 switch (alg) { 1470 case CRYPTO_SHA1: 1471 case CRYPTO_SHA1_HMAC: 1472 case CRYPTO_SHA2_256_HMAC: 1473 case CRYPTO_SHA2_384_HMAC: 1474 case CRYPTO_SHA2_512_HMAC: 1475 return (true); 1476 default: 1477 return (false); 1478 } 1479 } 1480 1481 static int 1482 sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode, 1483 u_int *hashlen) 1484 { 1485 1486 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1487 *eu = SEC_EU_NONE; 1488 1489 switch (csp->csp_auth_alg) { 1490 case CRYPTO_SHA1_HMAC: 1491 *mode |= SEC_MDEU_MODE_HMAC; 1492 /* FALLTHROUGH */ 1493 case CRYPTO_SHA1: 1494 *eu = SEC_EU_MDEU_A; 1495 *mode |= SEC_MDEU_MODE_SHA1; 1496 *hashlen = SHA1_HASH_LEN; 1497 break; 1498 case CRYPTO_SHA2_256_HMAC: 1499 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1500 *eu = SEC_EU_MDEU_A; 1501 break; 1502 case CRYPTO_SHA2_384_HMAC: 1503 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1504 *eu = SEC_EU_MDEU_B; 1505 break; 1506 case CRYPTO_SHA2_512_HMAC: 1507 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1508 *eu = SEC_EU_MDEU_B; 1509 break; 1510 default: 1511 return (EINVAL); 1512 } 1513 1514 if (*mode & SEC_MDEU_MODE_HMAC) 1515 *hashlen = SEC_HMAC_HASH_LEN; 1516 1517 return (0); 1518 } 1519 1520 static bool 1521 sec_mdeu_newsession(const struct crypto_session_params *csp) 1522 { 1523 1524 return (sec_mdeu_can_handle(csp->csp_auth_alg)); 1525 } 1526 1527 static int 1528 sec_mdeu_make_desc(struct sec_softc *sc, 1529 const struct crypto_session_params *csp, 1530 struct sec_desc *desc, struct cryptop *crp) 1531 { 1532 struct sec_hw_desc *hd = desc->sd_desc; 1533 u_int eu, mode, hashlen; 1534 int error; 1535 1536 error = sec_mdeu_config(csp, &eu, &mode, &hashlen); 1537 if (error) 1538 return (error); 1539 1540 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1541 hd->shd_eu_sel0 = eu; 1542 hd->shd_mode0 = mode; 1543 hd->shd_eu_sel1 = SEC_EU_NONE; 1544 hd->shd_mode1 = 0; 1545 1546 /* Pointer 0: NULL */ 1547 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1548 if (error) 1549 return (error); 1550 1551 /* Pointer 1: Context In (Not used: NULL) */ 1552 error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1553 if (error) 1554 return (error); 1555 1556 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1557 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1558 error = sec_make_pointer_direct(sc, desc, 2, 1559 desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1560 shd_mkey), csp->csp_auth_klen); 1561 else 1562 error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1563 1564 if (error) 1565 return (error); 1566 1567 /* Pointer 3: Input Data */ 1568 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, 1569 crp->crp_payload_length); 1570 if (error) 1571 return (error); 1572 1573 /* Pointer 4: NULL */ 1574 error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1575 if (error) 1576 return (error); 1577 1578 /* Pointer 5: Hash out */ 1579 error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr + 1580 offsetof(struct sec_hw_desc, shd_digest), hashlen); 1581 if (error) 1582 return (error); 1583 1584 /* Pointer 6: NULL */ 1585 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1586 1587 return (0); 1588 } 1589