1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 19 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 24 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 30 * 3.0 are supported. 31 */ 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/endian.h> 37 #include <sys/kernel.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/mutex.h> 43 #include <sys/random.h> 44 #include <sys/rman.h> 45 46 #include <machine/_inttypes.h> 47 #include <machine/bus.h> 48 #include <machine/resource.h> 49 50 #include <opencrypto/cryptodev.h> 51 #include <opencrypto/xform_auth.h> 52 #include "cryptodev_if.h" 53 54 #include <dev/ofw/ofw_bus_subr.h> 55 #include <dev/sec/sec.h> 56 57 static int sec_probe(device_t dev); 58 static int sec_attach(device_t dev); 59 static int sec_detach(device_t dev); 60 static int sec_suspend(device_t dev); 61 static int sec_resume(device_t dev); 62 static int sec_shutdown(device_t dev); 63 static void sec_primary_intr(void *arg); 64 static void sec_secondary_intr(void *arg); 65 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 66 void **ihand, int *irid, driver_intr_t handler, const char *iname); 67 static void sec_release_intr(struct sec_softc *sc, struct resource *ires, 68 void *ihand, int irid, const char *iname); 69 static int sec_controller_reset(struct sec_softc *sc); 70 static int sec_channel_reset(struct sec_softc *sc, int channel, int full); 71 static int sec_init(struct sec_softc *sc); 72 static int sec_alloc_dma_mem(struct sec_softc *sc, 73 struct sec_dma_mem *dma_mem, bus_size_t size); 74 static int sec_desc_map_dma(struct sec_softc *sc, 75 struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size, 76 struct sec_desc_map_info *sdmi); 77 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 78 static void sec_enqueue(struct sec_softc *sc); 79 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 80 int channel); 81 static int sec_eu_channel(struct sec_softc *sc, int eu); 82 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 83 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize); 84 static int sec_make_pointer_direct(struct sec_softc *sc, 85 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 86 static int sec_probesession(device_t dev, 87 const struct crypto_session_params *csp); 88 static int sec_newsession(device_t dev, crypto_session_t cses, 89 const struct crypto_session_params *csp); 90 static int sec_process(device_t dev, struct cryptop *crp, int hint); 91 static int sec_build_common_ns_desc(struct sec_softc *sc, 92 struct sec_desc *desc, const struct crypto_session_params *csp, 93 struct cryptop *crp); 94 static int sec_build_common_s_desc(struct sec_softc *sc, 95 struct sec_desc *desc, const struct crypto_session_params *csp, 96 struct cryptop *crp); 97 98 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 99 100 /* AESU */ 101 static bool sec_aesu_newsession(const struct crypto_session_params *csp); 102 static int sec_aesu_make_desc(struct sec_softc *sc, 103 const struct crypto_session_params *csp, struct sec_desc *desc, 104 struct cryptop *crp); 105 106 /* MDEU */ 107 static bool sec_mdeu_can_handle(u_int alg); 108 static int sec_mdeu_config(const struct crypto_session_params *csp, 109 u_int *eu, u_int *mode, u_int *hashlen); 110 static bool sec_mdeu_newsession(const struct crypto_session_params *csp); 111 static int sec_mdeu_make_desc(struct sec_softc *sc, 112 const struct crypto_session_params *csp, struct sec_desc *desc, 113 struct cryptop *crp); 114 115 static device_method_t sec_methods[] = { 116 /* Device interface */ 117 DEVMETHOD(device_probe, sec_probe), 118 DEVMETHOD(device_attach, sec_attach), 119 DEVMETHOD(device_detach, sec_detach), 120 121 DEVMETHOD(device_suspend, sec_suspend), 122 DEVMETHOD(device_resume, sec_resume), 123 DEVMETHOD(device_shutdown, sec_shutdown), 124 125 /* Crypto methods */ 126 DEVMETHOD(cryptodev_probesession, sec_probesession), 127 DEVMETHOD(cryptodev_newsession, sec_newsession), 128 DEVMETHOD(cryptodev_process, sec_process), 129 130 DEVMETHOD_END 131 }; 132 static driver_t sec_driver = { 133 "sec", 134 sec_methods, 135 sizeof(struct sec_softc), 136 }; 137 138 DRIVER_MODULE(sec, simplebus, sec_driver, 0, 0); 139 MODULE_DEPEND(sec, crypto, 1, 1, 1); 140 141 static struct sec_eu_methods sec_eus[] = { 142 { 143 sec_aesu_newsession, 144 sec_aesu_make_desc, 145 }, 146 { 147 sec_mdeu_newsession, 148 sec_mdeu_make_desc, 149 }, 150 { NULL, NULL } 151 }; 152 153 static inline void 154 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 155 { 156 157 /* Sync only if dma memory is valid */ 158 if (dma_mem->dma_vaddr != NULL) 159 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 160 } 161 162 static inline void * 163 sec_get_pointer_data(struct sec_desc *desc, u_int n) 164 { 165 166 return (desc->sd_ptr_dmem[n].dma_vaddr); 167 } 168 169 static int 170 sec_probe(device_t dev) 171 { 172 struct sec_softc *sc; 173 uint64_t id; 174 175 if (!ofw_bus_status_okay(dev)) 176 return (ENXIO); 177 178 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0")) 179 return (ENXIO); 180 181 sc = device_get_softc(dev); 182 183 sc->sc_rrid = 0; 184 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 185 RF_ACTIVE); 186 187 if (sc->sc_rres == NULL) 188 return (ENXIO); 189 190 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 191 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 192 193 id = SEC_READ(sc, SEC_ID); 194 195 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 196 197 switch (id) { 198 case SEC_20_ID: 199 device_set_desc(dev, "Freescale Security Engine 2.0"); 200 sc->sc_version = 2; 201 break; 202 case SEC_30_ID: 203 device_set_desc(dev, "Freescale Security Engine 3.0"); 204 sc->sc_version = 3; 205 break; 206 case SEC_31_ID: 207 device_set_desc(dev, "Freescale Security Engine 3.1"); 208 sc->sc_version = 3; 209 break; 210 default: 211 device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id); 212 return (ENXIO); 213 } 214 215 return (0); 216 } 217 218 static int 219 sec_attach(device_t dev) 220 { 221 struct sec_softc *sc; 222 struct sec_hw_lt *lt; 223 int error = 0; 224 int i; 225 226 sc = device_get_softc(dev); 227 sc->sc_dev = dev; 228 sc->sc_blocked = 0; 229 sc->sc_shutdown = 0; 230 231 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session), 232 CRYPTOCAP_F_HARDWARE); 233 if (sc->sc_cid < 0) { 234 device_printf(dev, "could not get crypto driver ID!\n"); 235 return (ENXIO); 236 } 237 238 /* Init locks */ 239 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 240 "SEC Controller lock", MTX_DEF); 241 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 242 "SEC Descriptors lock", MTX_DEF); 243 244 /* Allocate I/O memory for SEC registers */ 245 sc->sc_rrid = 0; 246 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 247 RF_ACTIVE); 248 249 if (sc->sc_rres == NULL) { 250 device_printf(dev, "could not allocate I/O memory!\n"); 251 goto fail1; 252 } 253 254 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 255 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 256 257 /* Setup interrupts */ 258 sc->sc_pri_irid = 0; 259 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 260 &sc->sc_pri_irid, sec_primary_intr, "primary"); 261 262 if (error) 263 goto fail2; 264 265 if (sc->sc_version == 3) { 266 sc->sc_sec_irid = 1; 267 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 268 &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 269 270 if (error) 271 goto fail3; 272 } 273 274 /* Alloc DMA memory for descriptors and link tables */ 275 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 276 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 277 278 if (error) 279 goto fail4; 280 281 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 282 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 283 284 if (error) 285 goto fail5; 286 287 /* Fill in descriptors and link tables */ 288 for (i = 0; i < SEC_DESCRIPTORS; i++) { 289 sc->sc_desc[i].sd_desc = 290 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 291 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 292 (i * sizeof(struct sec_hw_desc)); 293 } 294 295 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 296 sc->sc_lt[i].sl_lt = 297 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 298 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 299 (i * sizeof(struct sec_hw_lt)); 300 } 301 302 /* Last entry in link table is used to create a circle */ 303 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 304 lt->shl_length = 0; 305 lt->shl_r = 0; 306 lt->shl_n = 1; 307 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 308 309 /* Init descriptor and link table queues pointers */ 310 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 311 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 312 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 313 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 314 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 315 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 316 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 317 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 318 319 /* Create masks for fast checks */ 320 sc->sc_int_error_mask = 0; 321 for (i = 0; i < SEC_CHANNELS; i++) 322 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 323 324 switch (sc->sc_version) { 325 case 2: 326 sc->sc_channel_idle_mask = 327 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 328 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 329 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 330 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 331 break; 332 case 3: 333 sc->sc_channel_idle_mask = 334 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 335 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 336 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 337 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 338 break; 339 } 340 341 /* Init hardware */ 342 error = sec_init(sc); 343 344 if (error) 345 goto fail6; 346 347 return (0); 348 349 fail6: 350 sec_free_dma_mem(&(sc->sc_lt_dmem)); 351 fail5: 352 sec_free_dma_mem(&(sc->sc_desc_dmem)); 353 fail4: 354 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 355 sc->sc_sec_irid, "secondary"); 356 fail3: 357 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 358 sc->sc_pri_irid, "primary"); 359 fail2: 360 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 361 fail1: 362 mtx_destroy(&sc->sc_controller_lock); 363 mtx_destroy(&sc->sc_descriptors_lock); 364 365 return (ENXIO); 366 } 367 368 static int 369 sec_detach(device_t dev) 370 { 371 struct sec_softc *sc = device_get_softc(dev); 372 int i, error, timeout = SEC_TIMEOUT; 373 374 /* Prepare driver to shutdown */ 375 SEC_LOCK(sc, descriptors); 376 sc->sc_shutdown = 1; 377 SEC_UNLOCK(sc, descriptors); 378 379 /* Wait until all queued processing finishes */ 380 while (1) { 381 SEC_LOCK(sc, descriptors); 382 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 383 SEC_UNLOCK(sc, descriptors); 384 385 if (i == 0) 386 break; 387 388 if (timeout < 0) { 389 device_printf(dev, "queue flush timeout!\n"); 390 391 /* DMA can be still active - stop it */ 392 for (i = 0; i < SEC_CHANNELS; i++) 393 sec_channel_reset(sc, i, 1); 394 395 break; 396 } 397 398 timeout -= 1000; 399 DELAY(1000); 400 } 401 402 /* Disable interrupts */ 403 SEC_WRITE(sc, SEC_IER, 0); 404 405 /* Unregister from OCF */ 406 crypto_unregister_all(sc->sc_cid); 407 408 /* Free DMA memory */ 409 for (i = 0; i < SEC_DESCRIPTORS; i++) 410 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 411 412 sec_free_dma_mem(&(sc->sc_lt_dmem)); 413 sec_free_dma_mem(&(sc->sc_desc_dmem)); 414 415 /* Release interrupts */ 416 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 417 sc->sc_pri_irid, "primary"); 418 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 419 sc->sc_sec_irid, "secondary"); 420 421 /* Release memory */ 422 if (sc->sc_rres) { 423 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 424 sc->sc_rres); 425 if (error) 426 device_printf(dev, "bus_release_resource() failed for" 427 " I/O memory, error %d\n", error); 428 429 sc->sc_rres = NULL; 430 } 431 432 mtx_destroy(&sc->sc_controller_lock); 433 mtx_destroy(&sc->sc_descriptors_lock); 434 435 return (0); 436 } 437 438 static int 439 sec_suspend(device_t dev) 440 { 441 442 return (0); 443 } 444 445 static int 446 sec_resume(device_t dev) 447 { 448 449 return (0); 450 } 451 452 static int 453 sec_shutdown(device_t dev) 454 { 455 456 return (0); 457 } 458 459 static int 460 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 461 int *irid, driver_intr_t handler, const char *iname) 462 { 463 int error; 464 465 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 466 RF_ACTIVE); 467 468 if ((*ires) == NULL) { 469 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 470 return (ENXIO); 471 } 472 473 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 474 NULL, handler, sc, ihand); 475 476 if (error) { 477 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 478 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 479 device_printf(sc->sc_dev, "could not release %s IRQ\n", 480 iname); 481 482 (*ires) = NULL; 483 return (error); 484 } 485 486 return (0); 487 } 488 489 static void 490 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 491 int irid, const char *iname) 492 { 493 int error; 494 495 if (ires == NULL) 496 return; 497 498 error = bus_teardown_intr(sc->sc_dev, ires, ihand); 499 if (error) 500 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 501 " IRQ, error %d\n", iname, error); 502 503 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 504 if (error) 505 device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 506 " IRQ, error %d\n", iname, error); 507 } 508 509 static void 510 sec_primary_intr(void *arg) 511 { 512 struct sec_session *ses; 513 struct sec_softc *sc = arg; 514 struct sec_desc *desc; 515 struct cryptop *crp; 516 uint64_t isr; 517 uint8_t hash[HASH_MAX_LEN]; 518 int i, wakeup = 0; 519 520 SEC_LOCK(sc, controller); 521 522 /* Check for errors */ 523 isr = SEC_READ(sc, SEC_ISR); 524 if (isr & sc->sc_int_error_mask) { 525 /* Check each channel for error */ 526 for (i = 0; i < SEC_CHANNELS; i++) { 527 if ((isr & SEC_INT_CH_ERR(i)) == 0) 528 continue; 529 530 device_printf(sc->sc_dev, 531 "I/O error on channel %i!\n", i); 532 533 /* Find and mark problematic descriptor */ 534 desc = sec_find_desc(sc, SEC_READ(sc, 535 SEC_CHAN_CDPR(i))); 536 537 if (desc != NULL) 538 desc->sd_error = EIO; 539 540 /* Do partial channel reset */ 541 sec_channel_reset(sc, i, 0); 542 } 543 } 544 545 /* ACK interrupt */ 546 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 547 548 SEC_UNLOCK(sc, controller); 549 SEC_LOCK(sc, descriptors); 550 551 /* Handle processed descriptors */ 552 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 553 554 while (SEC_QUEUED_DESC_CNT(sc) > 0) { 555 desc = SEC_GET_QUEUED_DESC(sc); 556 557 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 558 SEC_PUT_BACK_QUEUED_DESC(sc); 559 break; 560 } 561 562 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 563 BUS_DMASYNC_PREWRITE); 564 565 crp = desc->sd_crp; 566 crp->crp_etype = desc->sd_error; 567 if (crp->crp_etype == 0) { 568 ses = crypto_get_driver_session(crp->crp_session); 569 if (ses->ss_mlen != 0) { 570 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 571 crypto_copydata(crp, 572 crp->crp_digest_start, 573 ses->ss_mlen, hash); 574 if (timingsafe_bcmp( 575 desc->sd_desc->shd_digest, 576 hash, ses->ss_mlen) != 0) 577 crp->crp_etype = EBADMSG; 578 } else 579 crypto_copyback(crp, 580 crp->crp_digest_start, 581 ses->ss_mlen, 582 desc->sd_desc->shd_digest); 583 } 584 } 585 crypto_done(desc->sd_crp); 586 587 SEC_DESC_FREE_POINTERS(desc); 588 SEC_DESC_FREE_LT(sc, desc); 589 SEC_DESC_QUEUED2FREE(sc); 590 } 591 592 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 593 594 if (!sc->sc_shutdown) { 595 wakeup = sc->sc_blocked; 596 sc->sc_blocked = 0; 597 } 598 599 SEC_UNLOCK(sc, descriptors); 600 601 /* Enqueue ready descriptors in hardware */ 602 sec_enqueue(sc); 603 604 if (wakeup) 605 crypto_unblock(sc->sc_cid, wakeup); 606 } 607 608 static void 609 sec_secondary_intr(void *arg) 610 { 611 struct sec_softc *sc = arg; 612 613 device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 614 sec_primary_intr(arg); 615 } 616 617 static int 618 sec_controller_reset(struct sec_softc *sc) 619 { 620 int timeout = SEC_TIMEOUT; 621 622 /* Reset Controller */ 623 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 624 625 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 626 DELAY(1000); 627 timeout -= 1000; 628 629 if (timeout < 0) { 630 device_printf(sc->sc_dev, "timeout while waiting for " 631 "device reset!\n"); 632 return (ETIMEDOUT); 633 } 634 } 635 636 return (0); 637 } 638 639 static int 640 sec_channel_reset(struct sec_softc *sc, int channel, int full) 641 { 642 int timeout = SEC_TIMEOUT; 643 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 644 uint64_t reg; 645 646 /* Reset Channel */ 647 reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 648 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 649 650 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 651 DELAY(1000); 652 timeout -= 1000; 653 654 if (timeout < 0) { 655 device_printf(sc->sc_dev, "timeout while waiting for " 656 "channel reset!\n"); 657 return (ETIMEDOUT); 658 } 659 } 660 661 if (full) { 662 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 663 664 switch(sc->sc_version) { 665 case 2: 666 reg |= SEC_CHAN_CCR_CDWE; 667 break; 668 case 3: 669 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 670 break; 671 } 672 673 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 674 } 675 676 return (0); 677 } 678 679 static int 680 sec_init(struct sec_softc *sc) 681 { 682 uint64_t reg; 683 int error, i; 684 685 /* Reset controller twice to clear all pending interrupts */ 686 error = sec_controller_reset(sc); 687 if (error) 688 return (error); 689 690 error = sec_controller_reset(sc); 691 if (error) 692 return (error); 693 694 /* Reset channels */ 695 for (i = 0; i < SEC_CHANNELS; i++) { 696 error = sec_channel_reset(sc, i, 1); 697 if (error) 698 return (error); 699 } 700 701 /* Enable Interrupts */ 702 reg = SEC_INT_ITO; 703 for (i = 0; i < SEC_CHANNELS; i++) 704 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 705 706 SEC_WRITE(sc, SEC_IER, reg); 707 708 return (error); 709 } 710 711 static void 712 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 713 { 714 struct sec_dma_mem *dma_mem = arg; 715 716 if (error) 717 return; 718 719 KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 720 dma_mem->dma_paddr = segs->ds_addr; 721 } 722 723 static void 724 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 725 int error) 726 { 727 struct sec_desc_map_info *sdmi = arg; 728 struct sec_softc *sc = sdmi->sdmi_sc; 729 struct sec_lt *lt = NULL; 730 bus_addr_t addr; 731 bus_size_t size; 732 int i; 733 734 SEC_LOCK_ASSERT(sc, descriptors); 735 736 if (error) 737 return; 738 739 for (i = 0; i < nseg; i++) { 740 addr = segs[i].ds_addr; 741 size = segs[i].ds_len; 742 743 /* Skip requested offset */ 744 if (sdmi->sdmi_offset >= size) { 745 sdmi->sdmi_offset -= size; 746 continue; 747 } 748 749 addr += sdmi->sdmi_offset; 750 size -= sdmi->sdmi_offset; 751 sdmi->sdmi_offset = 0; 752 753 /* Do not link more than requested */ 754 if (sdmi->sdmi_size < size) 755 size = sdmi->sdmi_size; 756 757 lt = SEC_ALLOC_LT_ENTRY(sc); 758 lt->sl_lt->shl_length = size; 759 lt->sl_lt->shl_r = 0; 760 lt->sl_lt->shl_n = 0; 761 lt->sl_lt->shl_ptr = addr; 762 763 if (sdmi->sdmi_lt_first == NULL) 764 sdmi->sdmi_lt_first = lt; 765 766 sdmi->sdmi_lt_used += 1; 767 768 if ((sdmi->sdmi_size -= size) == 0) 769 break; 770 } 771 772 sdmi->sdmi_lt_last = lt; 773 } 774 775 static int 776 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 777 bus_size_t size) 778 { 779 int error; 780 781 if (dma_mem->dma_vaddr != NULL) 782 return (EBUSY); 783 784 error = bus_dma_tag_create(NULL, /* parent */ 785 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 786 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 787 BUS_SPACE_MAXADDR, /* highaddr */ 788 NULL, NULL, /* filtfunc, filtfuncarg */ 789 size, 1, /* maxsize, nsegments */ 790 size, 0, /* maxsegsz, flags */ 791 NULL, NULL, /* lockfunc, lockfuncarg */ 792 &(dma_mem->dma_tag)); /* dmat */ 793 794 if (error) { 795 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 796 " %i!\n", error); 797 goto err1; 798 } 799 800 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 801 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 802 803 if (error) { 804 device_printf(sc->sc_dev, "failed to allocate DMA safe" 805 " memory, error %i!\n", error); 806 goto err2; 807 } 808 809 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 810 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 811 BUS_DMA_NOWAIT); 812 813 if (error) { 814 device_printf(sc->sc_dev, "cannot get address of the DMA" 815 " memory, error %i\n", error); 816 goto err3; 817 } 818 819 dma_mem->dma_is_map = 0; 820 return (0); 821 822 err3: 823 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 824 err2: 825 bus_dma_tag_destroy(dma_mem->dma_tag); 826 err1: 827 dma_mem->dma_vaddr = NULL; 828 return(error); 829 } 830 831 static int 832 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 833 struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi) 834 { 835 int error; 836 837 if (dma_mem->dma_vaddr != NULL) 838 return (EBUSY); 839 840 switch (crp->crp_buf.cb_type) { 841 case CRYPTO_BUF_CONTIG: 842 break; 843 case CRYPTO_BUF_UIO: 844 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 845 break; 846 case CRYPTO_BUF_MBUF: 847 size = m_length(crp->crp_buf.cb_mbuf, NULL); 848 break; 849 case CRYPTO_BUF_SINGLE_MBUF: 850 size = crp->crp_buf.cb_mbuf->m_len; 851 break; 852 case CRYPTO_BUF_VMPAGE: 853 size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset; 854 break; 855 default: 856 return (EINVAL); 857 } 858 859 error = bus_dma_tag_create(NULL, /* parent */ 860 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 861 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 862 BUS_SPACE_MAXADDR, /* highaddr */ 863 NULL, NULL, /* filtfunc, filtfuncarg */ 864 size, /* maxsize */ 865 SEC_FREE_LT_CNT(sc), /* nsegments */ 866 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 867 NULL, NULL, /* lockfunc, lockfuncarg */ 868 &(dma_mem->dma_tag)); /* dmat */ 869 870 if (error) { 871 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 872 " %i!\n", error); 873 dma_mem->dma_vaddr = NULL; 874 return (error); 875 } 876 877 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 878 879 if (error) { 880 device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 881 "\n", error); 882 bus_dma_tag_destroy(dma_mem->dma_tag); 883 return (error); 884 } 885 886 error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp, 887 sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 888 889 if (error) { 890 device_printf(sc->sc_dev, "cannot get address of the DMA" 891 " memory, error %i!\n", error); 892 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 893 bus_dma_tag_destroy(dma_mem->dma_tag); 894 return (error); 895 } 896 897 dma_mem->dma_is_map = 1; 898 dma_mem->dma_vaddr = crp; 899 900 return (0); 901 } 902 903 static void 904 sec_free_dma_mem(struct sec_dma_mem *dma_mem) 905 { 906 907 /* Check for double free */ 908 if (dma_mem->dma_vaddr == NULL) 909 return; 910 911 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 912 913 if (dma_mem->dma_is_map) 914 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 915 else 916 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 917 dma_mem->dma_map); 918 919 bus_dma_tag_destroy(dma_mem->dma_tag); 920 dma_mem->dma_vaddr = NULL; 921 } 922 923 static int 924 sec_eu_channel(struct sec_softc *sc, int eu) 925 { 926 uint64_t reg; 927 int channel = 0; 928 929 SEC_LOCK_ASSERT(sc, controller); 930 931 reg = SEC_READ(sc, SEC_EUASR); 932 933 switch (eu) { 934 case SEC_EU_AFEU: 935 channel = SEC_EUASR_AFEU(reg); 936 break; 937 case SEC_EU_DEU: 938 channel = SEC_EUASR_DEU(reg); 939 break; 940 case SEC_EU_MDEU_A: 941 case SEC_EU_MDEU_B: 942 channel = SEC_EUASR_MDEU(reg); 943 break; 944 case SEC_EU_RNGU: 945 channel = SEC_EUASR_RNGU(reg); 946 break; 947 case SEC_EU_PKEU: 948 channel = SEC_EUASR_PKEU(reg); 949 break; 950 case SEC_EU_AESU: 951 channel = SEC_EUASR_AESU(reg); 952 break; 953 case SEC_EU_KEU: 954 channel = SEC_EUASR_KEU(reg); 955 break; 956 case SEC_EU_CRCU: 957 channel = SEC_EUASR_CRCU(reg); 958 break; 959 } 960 961 return (channel - 1); 962 } 963 964 static int 965 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 966 { 967 u_int fflvl = SEC_MAX_FIFO_LEVEL; 968 uint64_t reg; 969 int i; 970 971 SEC_LOCK_ASSERT(sc, controller); 972 973 /* Find free channel if have not got one */ 974 if (channel < 0) { 975 for (i = 0; i < SEC_CHANNELS; i++) { 976 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 977 978 if ((reg & sc->sc_channel_idle_mask) == 0) { 979 channel = i; 980 break; 981 } 982 } 983 } 984 985 /* There is no free channel */ 986 if (channel < 0) 987 return (-1); 988 989 /* Check FIFO level on selected channel */ 990 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 991 992 switch(sc->sc_version) { 993 case 2: 994 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 995 break; 996 case 3: 997 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 998 break; 999 } 1000 1001 if (fflvl >= SEC_MAX_FIFO_LEVEL) 1002 return (-1); 1003 1004 /* Enqueue descriptor in channel */ 1005 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1006 1007 return (channel); 1008 } 1009 1010 static void 1011 sec_enqueue(struct sec_softc *sc) 1012 { 1013 struct sec_desc *desc; 1014 int ch0, ch1; 1015 1016 SEC_LOCK(sc, descriptors); 1017 SEC_LOCK(sc, controller); 1018 1019 while (SEC_READY_DESC_CNT(sc) > 0) { 1020 desc = SEC_GET_READY_DESC(sc); 1021 1022 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1023 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1024 1025 /* 1026 * Both EU are used by the same channel. 1027 * Enqueue descriptor in channel used by busy EUs. 1028 */ 1029 if (ch0 >= 0 && ch0 == ch1) { 1030 if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1031 SEC_DESC_READY2QUEUED(sc); 1032 continue; 1033 } 1034 } 1035 1036 /* 1037 * Only one EU is free. 1038 * Enqueue descriptor in channel used by busy EU. 1039 */ 1040 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1041 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1042 >= 0) { 1043 SEC_DESC_READY2QUEUED(sc); 1044 continue; 1045 } 1046 } 1047 1048 /* 1049 * Both EU are free. 1050 * Enqueue descriptor in first free channel. 1051 */ 1052 if (ch0 < 0 && ch1 < 0) { 1053 if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1054 SEC_DESC_READY2QUEUED(sc); 1055 continue; 1056 } 1057 } 1058 1059 /* Current descriptor can not be queued at the moment */ 1060 SEC_PUT_BACK_READY_DESC(sc); 1061 break; 1062 } 1063 1064 SEC_UNLOCK(sc, controller); 1065 SEC_UNLOCK(sc, descriptors); 1066 } 1067 1068 static struct sec_desc * 1069 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1070 { 1071 struct sec_desc *desc = NULL; 1072 int i; 1073 1074 SEC_LOCK_ASSERT(sc, descriptors); 1075 1076 for (i = 0; i < SEC_CHANNELS; i++) { 1077 if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1078 desc = &(sc->sc_desc[i]); 1079 break; 1080 } 1081 } 1082 1083 return (desc); 1084 } 1085 1086 static int 1087 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1088 bus_addr_t data, bus_size_t dsize) 1089 { 1090 struct sec_hw_desc_ptr *ptr; 1091 1092 SEC_LOCK_ASSERT(sc, descriptors); 1093 1094 ptr = &(desc->sd_desc->shd_pointer[n]); 1095 ptr->shdp_length = dsize; 1096 ptr->shdp_extent = 0; 1097 ptr->shdp_j = 0; 1098 ptr->shdp_ptr = data; 1099 1100 return (0); 1101 } 1102 1103 static int 1104 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1105 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize) 1106 { 1107 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1108 struct sec_hw_desc_ptr *ptr; 1109 int error; 1110 1111 SEC_LOCK_ASSERT(sc, descriptors); 1112 1113 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize, 1114 &sdmi); 1115 1116 if (error) 1117 return (error); 1118 1119 sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1120 desc->sd_lt_used += sdmi.sdmi_lt_used; 1121 1122 ptr = &(desc->sd_desc->shd_pointer[n]); 1123 ptr->shdp_length = dsize; 1124 ptr->shdp_extent = 0; 1125 ptr->shdp_j = 1; 1126 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1127 1128 return (0); 1129 } 1130 1131 static bool 1132 sec_cipher_supported(const struct crypto_session_params *csp) 1133 { 1134 1135 switch (csp->csp_cipher_alg) { 1136 case CRYPTO_AES_CBC: 1137 /* AESU */ 1138 if (csp->csp_ivlen != AES_BLOCK_LEN) 1139 return (false); 1140 break; 1141 default: 1142 return (false); 1143 } 1144 1145 if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN) 1146 return (false); 1147 1148 return (true); 1149 } 1150 1151 static bool 1152 sec_auth_supported(struct sec_softc *sc, 1153 const struct crypto_session_params *csp) 1154 { 1155 1156 switch (csp->csp_auth_alg) { 1157 case CRYPTO_SHA2_384_HMAC: 1158 case CRYPTO_SHA2_512_HMAC: 1159 if (sc->sc_version < 3) 1160 return (false); 1161 /* FALLTHROUGH */ 1162 case CRYPTO_SHA1_HMAC: 1163 case CRYPTO_SHA2_256_HMAC: 1164 if (csp->csp_auth_klen > SEC_MAX_KEY_LEN) 1165 return (false); 1166 break; 1167 case CRYPTO_SHA1: 1168 break; 1169 default: 1170 return (false); 1171 } 1172 return (true); 1173 } 1174 1175 static int 1176 sec_probesession(device_t dev, const struct crypto_session_params *csp) 1177 { 1178 struct sec_softc *sc = device_get_softc(dev); 1179 1180 if (csp->csp_flags != 0) 1181 return (EINVAL); 1182 switch (csp->csp_mode) { 1183 case CSP_MODE_DIGEST: 1184 if (!sec_auth_supported(sc, csp)) 1185 return (EINVAL); 1186 break; 1187 case CSP_MODE_CIPHER: 1188 if (!sec_cipher_supported(csp)) 1189 return (EINVAL); 1190 break; 1191 case CSP_MODE_ETA: 1192 if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp)) 1193 return (EINVAL); 1194 break; 1195 default: 1196 return (EINVAL); 1197 } 1198 return (CRYPTODEV_PROBE_HARDWARE); 1199 } 1200 1201 static int 1202 sec_newsession(device_t dev, crypto_session_t cses, 1203 const struct crypto_session_params *csp) 1204 { 1205 struct sec_eu_methods *eu = sec_eus; 1206 struct sec_session *ses; 1207 1208 ses = crypto_get_driver_session(cses); 1209 1210 /* Find EU for this session */ 1211 while (eu->sem_make_desc != NULL) { 1212 if (eu->sem_newsession(csp)) 1213 break; 1214 eu++; 1215 } 1216 KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session")); 1217 1218 /* Save cipher key */ 1219 if (csp->csp_cipher_key != NULL) 1220 memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen); 1221 1222 /* Save digest key */ 1223 if (csp->csp_auth_key != NULL) 1224 memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen); 1225 1226 if (csp->csp_auth_alg != 0) { 1227 if (csp->csp_auth_mlen == 0) 1228 ses->ss_mlen = crypto_auth_hash(csp)->hashsize; 1229 else 1230 ses->ss_mlen = csp->csp_auth_mlen; 1231 } 1232 1233 return (0); 1234 } 1235 1236 static int 1237 sec_process(device_t dev, struct cryptop *crp, int hint) 1238 { 1239 struct sec_softc *sc = device_get_softc(dev); 1240 struct sec_desc *desc = NULL; 1241 const struct crypto_session_params *csp; 1242 struct sec_session *ses; 1243 int error = 0; 1244 1245 ses = crypto_get_driver_session(crp->crp_session); 1246 csp = crypto_get_params(crp->crp_session); 1247 1248 /* Check for input length */ 1249 if (crypto_buffer_len(&crp->crp_buf) > SEC_MAX_DMA_BLOCK_SIZE) { 1250 crp->crp_etype = E2BIG; 1251 crypto_done(crp); 1252 return (0); 1253 } 1254 1255 SEC_LOCK(sc, descriptors); 1256 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1257 1258 /* Block driver if there is no free descriptors or we are going down */ 1259 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1260 sc->sc_blocked |= CRYPTO_SYMQ; 1261 SEC_UNLOCK(sc, descriptors); 1262 return (ERESTART); 1263 } 1264 1265 /* Prepare descriptor */ 1266 desc = SEC_GET_FREE_DESC(sc); 1267 desc->sd_lt_used = 0; 1268 desc->sd_error = 0; 1269 desc->sd_crp = crp; 1270 1271 if (csp->csp_cipher_alg != 0) 1272 crypto_read_iv(crp, desc->sd_desc->shd_iv); 1273 1274 if (crp->crp_cipher_key != NULL) 1275 memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen); 1276 1277 if (crp->crp_auth_key != NULL) 1278 memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen); 1279 1280 memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen); 1281 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen); 1282 1283 error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp); 1284 1285 if (error) { 1286 SEC_DESC_FREE_POINTERS(desc); 1287 SEC_DESC_PUT_BACK_LT(sc, desc); 1288 SEC_PUT_BACK_FREE_DESC(sc); 1289 SEC_UNLOCK(sc, descriptors); 1290 crp->crp_etype = error; 1291 crypto_done(crp); 1292 return (0); 1293 } 1294 1295 /* 1296 * Skip DONE interrupt if this is not last request in burst, but only 1297 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1298 * signaling on each descriptor. 1299 */ 1300 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1301 desc->sd_desc->shd_dn = 0; 1302 else 1303 desc->sd_desc->shd_dn = 1; 1304 1305 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1306 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1307 BUS_DMASYNC_POSTWRITE); 1308 SEC_DESC_FREE2READY(sc); 1309 SEC_UNLOCK(sc, descriptors); 1310 1311 /* Enqueue ready descriptors in hardware */ 1312 sec_enqueue(sc); 1313 1314 return (0); 1315 } 1316 1317 static int 1318 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1319 const struct crypto_session_params *csp, struct cryptop *crp) 1320 { 1321 struct sec_hw_desc *hd = desc->sd_desc; 1322 int error; 1323 1324 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1325 hd->shd_eu_sel1 = SEC_EU_NONE; 1326 hd->shd_mode1 = 0; 1327 1328 /* Pointer 0: NULL */ 1329 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1330 if (error) 1331 return (error); 1332 1333 /* Pointer 1: IV IN */ 1334 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1335 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); 1336 if (error) 1337 return (error); 1338 1339 /* Pointer 2: Cipher Key */ 1340 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1341 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); 1342 if (error) 1343 return (error); 1344 1345 /* Pointer 3: Data IN */ 1346 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, 1347 crp->crp_payload_length); 1348 if (error) 1349 return (error); 1350 1351 /* Pointer 4: Data OUT */ 1352 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, 1353 crp->crp_payload_length); 1354 if (error) 1355 return (error); 1356 1357 /* Pointer 5: IV OUT (Not used: NULL) */ 1358 error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1359 if (error) 1360 return (error); 1361 1362 /* Pointer 6: NULL */ 1363 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1364 1365 return (error); 1366 } 1367 1368 static int 1369 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1370 const struct crypto_session_params *csp, struct cryptop *crp) 1371 { 1372 struct sec_hw_desc *hd = desc->sd_desc; 1373 u_int eu, mode, hashlen; 1374 int error; 1375 1376 error = sec_mdeu_config(csp, &eu, &mode, &hashlen); 1377 if (error) 1378 return (error); 1379 1380 hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1381 hd->shd_eu_sel1 = eu; 1382 hd->shd_mode1 = mode; 1383 1384 /* Pointer 0: HMAC Key */ 1385 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1386 offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen); 1387 if (error) 1388 return (error); 1389 1390 /* Pointer 1: HMAC-Only Data IN */ 1391 error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start, 1392 crp->crp_aad_length); 1393 if (error) 1394 return (error); 1395 1396 /* Pointer 2: Cipher Key */ 1397 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1398 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); 1399 if (error) 1400 return (error); 1401 1402 /* Pointer 3: IV IN */ 1403 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1404 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); 1405 if (error) 1406 return (error); 1407 1408 /* Pointer 4: Data IN */ 1409 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, 1410 crp->crp_payload_length); 1411 if (error) 1412 return (error); 1413 1414 /* Pointer 5: Data OUT */ 1415 error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start, 1416 crp->crp_payload_length); 1417 if (error) 1418 return (error); 1419 1420 /* Pointer 6: HMAC OUT */ 1421 error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr + 1422 offsetof(struct sec_hw_desc, shd_digest), hashlen); 1423 1424 return (error); 1425 } 1426 1427 /* AESU */ 1428 1429 static bool 1430 sec_aesu_newsession(const struct crypto_session_params *csp) 1431 { 1432 1433 return (csp->csp_cipher_alg == CRYPTO_AES_CBC); 1434 } 1435 1436 static int 1437 sec_aesu_make_desc(struct sec_softc *sc, 1438 const struct crypto_session_params *csp, struct sec_desc *desc, 1439 struct cryptop *crp) 1440 { 1441 struct sec_hw_desc *hd = desc->sd_desc; 1442 int error; 1443 1444 hd->shd_eu_sel0 = SEC_EU_AESU; 1445 hd->shd_mode0 = SEC_AESU_MODE_CBC; 1446 1447 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1448 hd->shd_mode0 |= SEC_AESU_MODE_ED; 1449 hd->shd_dir = 0; 1450 } else 1451 hd->shd_dir = 1; 1452 1453 if (csp->csp_mode == CSP_MODE_ETA) 1454 error = sec_build_common_s_desc(sc, desc, csp, crp); 1455 else 1456 error = sec_build_common_ns_desc(sc, desc, csp, crp); 1457 1458 return (error); 1459 } 1460 1461 /* MDEU */ 1462 1463 static bool 1464 sec_mdeu_can_handle(u_int alg) 1465 { 1466 switch (alg) { 1467 case CRYPTO_SHA1: 1468 case CRYPTO_SHA1_HMAC: 1469 case CRYPTO_SHA2_256_HMAC: 1470 case CRYPTO_SHA2_384_HMAC: 1471 case CRYPTO_SHA2_512_HMAC: 1472 return (true); 1473 default: 1474 return (false); 1475 } 1476 } 1477 1478 static int 1479 sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode, 1480 u_int *hashlen) 1481 { 1482 1483 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1484 *eu = SEC_EU_NONE; 1485 1486 switch (csp->csp_auth_alg) { 1487 case CRYPTO_SHA1_HMAC: 1488 *mode |= SEC_MDEU_MODE_HMAC; 1489 /* FALLTHROUGH */ 1490 case CRYPTO_SHA1: 1491 *eu = SEC_EU_MDEU_A; 1492 *mode |= SEC_MDEU_MODE_SHA1; 1493 *hashlen = SHA1_HASH_LEN; 1494 break; 1495 case CRYPTO_SHA2_256_HMAC: 1496 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1497 *eu = SEC_EU_MDEU_A; 1498 break; 1499 case CRYPTO_SHA2_384_HMAC: 1500 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1501 *eu = SEC_EU_MDEU_B; 1502 break; 1503 case CRYPTO_SHA2_512_HMAC: 1504 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1505 *eu = SEC_EU_MDEU_B; 1506 break; 1507 default: 1508 return (EINVAL); 1509 } 1510 1511 if (*mode & SEC_MDEU_MODE_HMAC) 1512 *hashlen = SEC_HMAC_HASH_LEN; 1513 1514 return (0); 1515 } 1516 1517 static bool 1518 sec_mdeu_newsession(const struct crypto_session_params *csp) 1519 { 1520 1521 return (sec_mdeu_can_handle(csp->csp_auth_alg)); 1522 } 1523 1524 static int 1525 sec_mdeu_make_desc(struct sec_softc *sc, 1526 const struct crypto_session_params *csp, 1527 struct sec_desc *desc, struct cryptop *crp) 1528 { 1529 struct sec_hw_desc *hd = desc->sd_desc; 1530 u_int eu, mode, hashlen; 1531 int error; 1532 1533 error = sec_mdeu_config(csp, &eu, &mode, &hashlen); 1534 if (error) 1535 return (error); 1536 1537 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1538 hd->shd_eu_sel0 = eu; 1539 hd->shd_mode0 = mode; 1540 hd->shd_eu_sel1 = SEC_EU_NONE; 1541 hd->shd_mode1 = 0; 1542 1543 /* Pointer 0: NULL */ 1544 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1545 if (error) 1546 return (error); 1547 1548 /* Pointer 1: Context In (Not used: NULL) */ 1549 error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1550 if (error) 1551 return (error); 1552 1553 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1554 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1555 error = sec_make_pointer_direct(sc, desc, 2, 1556 desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1557 shd_mkey), csp->csp_auth_klen); 1558 else 1559 error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1560 1561 if (error) 1562 return (error); 1563 1564 /* Pointer 3: Input Data */ 1565 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, 1566 crp->crp_payload_length); 1567 if (error) 1568 return (error); 1569 1570 /* Pointer 4: NULL */ 1571 error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1572 if (error) 1573 return (error); 1574 1575 /* Pointer 5: Hash out */ 1576 error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr + 1577 offsetof(struct sec_hw_desc, shd_digest), hashlen); 1578 if (error) 1579 return (error); 1580 1581 /* Pointer 6: NULL */ 1582 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1583 1584 return (0); 1585 } 1586