1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 19 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 24 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 30 * 3.0 are supported. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/bus.h> 39 #include <sys/endian.h> 40 #include <sys/kernel.h> 41 #include <sys/lock.h> 42 #include <sys/malloc.h> 43 #include <sys/mbuf.h> 44 #include <sys/module.h> 45 #include <sys/mutex.h> 46 #include <sys/random.h> 47 #include <sys/rman.h> 48 49 #include <machine/_inttypes.h> 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include "cryptodev_if.h" 55 56 #include <dev/ofw/ofw_bus_subr.h> 57 #include <dev/sec/sec.h> 58 59 static int sec_probe(device_t dev); 60 static int sec_attach(device_t dev); 61 static int sec_detach(device_t dev); 62 static int sec_suspend(device_t dev); 63 static int sec_resume(device_t dev); 64 static int sec_shutdown(device_t dev); 65 static void sec_primary_intr(void *arg); 66 static void sec_secondary_intr(void *arg); 67 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 68 void **ihand, int *irid, driver_intr_t handler, const char *iname); 69 static void sec_release_intr(struct sec_softc *sc, struct resource *ires, 70 void *ihand, int irid, const char *iname); 71 static int sec_controller_reset(struct sec_softc *sc); 72 static int sec_channel_reset(struct sec_softc *sc, int channel, int full); 73 static int sec_init(struct sec_softc *sc); 74 static int sec_alloc_dma_mem(struct sec_softc *sc, 75 struct sec_dma_mem *dma_mem, bus_size_t size); 76 static int sec_desc_map_dma(struct sec_softc *sc, 77 struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type, 78 struct sec_desc_map_info *sdmi); 79 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 80 static void sec_enqueue(struct sec_softc *sc); 81 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 82 int channel); 83 static int sec_eu_channel(struct sec_softc *sc, int eu); 84 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 85 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype); 86 static int sec_make_pointer_direct(struct sec_softc *sc, 87 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 88 static int sec_newsession(device_t dev, crypto_session_t cses, 89 struct cryptoini *cri); 90 static int sec_process(device_t dev, struct cryptop *crp, int hint); 91 static int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 92 struct cryptoini **mac); 93 static int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 94 struct cryptodesc **mac); 95 static int sec_build_common_ns_desc(struct sec_softc *sc, 96 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 97 struct cryptodesc *enc, int buftype); 98 static int sec_build_common_s_desc(struct sec_softc *sc, 99 struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 100 struct cryptodesc *enc, struct cryptodesc *mac, int buftype); 101 102 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 103 104 /* AESU */ 105 static int sec_aesu_newsession(struct sec_softc *sc, 106 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 107 static int sec_aesu_make_desc(struct sec_softc *sc, 108 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 109 int buftype); 110 111 /* DEU */ 112 static int sec_deu_newsession(struct sec_softc *sc, 113 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 114 static int sec_deu_make_desc(struct sec_softc *sc, 115 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 116 int buftype); 117 118 /* MDEU */ 119 static int sec_mdeu_can_handle(u_int alg); 120 static int sec_mdeu_config(struct cryptodesc *crd, 121 u_int *eu, u_int *mode, u_int *hashlen); 122 static int sec_mdeu_newsession(struct sec_softc *sc, 123 struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 124 static int sec_mdeu_make_desc(struct sec_softc *sc, 125 struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 126 int buftype); 127 128 static device_method_t sec_methods[] = { 129 /* Device interface */ 130 DEVMETHOD(device_probe, sec_probe), 131 DEVMETHOD(device_attach, sec_attach), 132 DEVMETHOD(device_detach, sec_detach), 133 134 DEVMETHOD(device_suspend, sec_suspend), 135 DEVMETHOD(device_resume, sec_resume), 136 DEVMETHOD(device_shutdown, sec_shutdown), 137 138 /* Crypto methods */ 139 DEVMETHOD(cryptodev_newsession, sec_newsession), 140 DEVMETHOD(cryptodev_process, sec_process), 141 142 DEVMETHOD_END 143 }; 144 static driver_t sec_driver = { 145 "sec", 146 sec_methods, 147 sizeof(struct sec_softc), 148 }; 149 150 static devclass_t sec_devclass; 151 DRIVER_MODULE(sec, simplebus, sec_driver, sec_devclass, 0, 0); 152 MODULE_DEPEND(sec, crypto, 1, 1, 1); 153 154 static struct sec_eu_methods sec_eus[] = { 155 { 156 sec_aesu_newsession, 157 sec_aesu_make_desc, 158 }, 159 { 160 sec_deu_newsession, 161 sec_deu_make_desc, 162 }, 163 { 164 sec_mdeu_newsession, 165 sec_mdeu_make_desc, 166 }, 167 { NULL, NULL } 168 }; 169 170 static inline void 171 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 172 { 173 174 /* Sync only if dma memory is valid */ 175 if (dma_mem->dma_vaddr != NULL) 176 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 177 } 178 179 static inline void * 180 sec_get_pointer_data(struct sec_desc *desc, u_int n) 181 { 182 183 return (desc->sd_ptr_dmem[n].dma_vaddr); 184 } 185 186 static int 187 sec_probe(device_t dev) 188 { 189 struct sec_softc *sc; 190 uint64_t id; 191 192 if (!ofw_bus_status_okay(dev)) 193 return (ENXIO); 194 195 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0")) 196 return (ENXIO); 197 198 sc = device_get_softc(dev); 199 200 sc->sc_rrid = 0; 201 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 202 RF_ACTIVE); 203 204 if (sc->sc_rres == NULL) 205 return (ENXIO); 206 207 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 208 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 209 210 id = SEC_READ(sc, SEC_ID); 211 212 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 213 214 switch (id) { 215 case SEC_20_ID: 216 device_set_desc(dev, "Freescale Security Engine 2.0"); 217 sc->sc_version = 2; 218 break; 219 case SEC_30_ID: 220 device_set_desc(dev, "Freescale Security Engine 3.0"); 221 sc->sc_version = 3; 222 break; 223 case SEC_31_ID: 224 device_set_desc(dev, "Freescale Security Engine 3.1"); 225 sc->sc_version = 3; 226 break; 227 default: 228 device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id); 229 return (ENXIO); 230 } 231 232 return (0); 233 } 234 235 static int 236 sec_attach(device_t dev) 237 { 238 struct sec_softc *sc; 239 struct sec_hw_lt *lt; 240 int error = 0; 241 int i; 242 243 sc = device_get_softc(dev); 244 sc->sc_dev = dev; 245 sc->sc_blocked = 0; 246 sc->sc_shutdown = 0; 247 248 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session), 249 CRYPTOCAP_F_HARDWARE); 250 if (sc->sc_cid < 0) { 251 device_printf(dev, "could not get crypto driver ID!\n"); 252 return (ENXIO); 253 } 254 255 /* Init locks */ 256 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 257 "SEC Controller lock", MTX_DEF); 258 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 259 "SEC Descriptors lock", MTX_DEF); 260 261 /* Allocate I/O memory for SEC registers */ 262 sc->sc_rrid = 0; 263 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 264 RF_ACTIVE); 265 266 if (sc->sc_rres == NULL) { 267 device_printf(dev, "could not allocate I/O memory!\n"); 268 goto fail1; 269 } 270 271 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 272 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 273 274 /* Setup interrupts */ 275 sc->sc_pri_irid = 0; 276 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 277 &sc->sc_pri_irid, sec_primary_intr, "primary"); 278 279 if (error) 280 goto fail2; 281 282 283 if (sc->sc_version == 3) { 284 sc->sc_sec_irid = 1; 285 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 286 &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 287 288 if (error) 289 goto fail3; 290 } 291 292 /* Alloc DMA memory for descriptors and link tables */ 293 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 294 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 295 296 if (error) 297 goto fail4; 298 299 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 300 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 301 302 if (error) 303 goto fail5; 304 305 /* Fill in descriptors and link tables */ 306 for (i = 0; i < SEC_DESCRIPTORS; i++) { 307 sc->sc_desc[i].sd_desc = 308 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 309 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 310 (i * sizeof(struct sec_hw_desc)); 311 } 312 313 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 314 sc->sc_lt[i].sl_lt = 315 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 316 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 317 (i * sizeof(struct sec_hw_lt)); 318 } 319 320 /* Last entry in link table is used to create a circle */ 321 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 322 lt->shl_length = 0; 323 lt->shl_r = 0; 324 lt->shl_n = 1; 325 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 326 327 /* Init descriptor and link table queues pointers */ 328 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 329 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 330 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 331 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 332 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 333 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 334 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 335 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 336 337 /* Create masks for fast checks */ 338 sc->sc_int_error_mask = 0; 339 for (i = 0; i < SEC_CHANNELS; i++) 340 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 341 342 switch (sc->sc_version) { 343 case 2: 344 sc->sc_channel_idle_mask = 345 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 346 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 347 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 348 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 349 break; 350 case 3: 351 sc->sc_channel_idle_mask = 352 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 353 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 354 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 355 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 356 break; 357 } 358 359 /* Init hardware */ 360 error = sec_init(sc); 361 362 if (error) 363 goto fail6; 364 365 /* Register in OCF (AESU) */ 366 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 367 368 /* Register in OCF (DEU) */ 369 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 370 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 371 372 /* Register in OCF (MDEU) */ 373 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 374 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 375 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 376 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 377 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 378 if (sc->sc_version >= 3) { 379 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 380 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 381 } 382 383 return (0); 384 385 fail6: 386 sec_free_dma_mem(&(sc->sc_lt_dmem)); 387 fail5: 388 sec_free_dma_mem(&(sc->sc_desc_dmem)); 389 fail4: 390 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 391 sc->sc_sec_irid, "secondary"); 392 fail3: 393 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 394 sc->sc_pri_irid, "primary"); 395 fail2: 396 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 397 fail1: 398 mtx_destroy(&sc->sc_controller_lock); 399 mtx_destroy(&sc->sc_descriptors_lock); 400 401 return (ENXIO); 402 } 403 404 static int 405 sec_detach(device_t dev) 406 { 407 struct sec_softc *sc = device_get_softc(dev); 408 int i, error, timeout = SEC_TIMEOUT; 409 410 /* Prepare driver to shutdown */ 411 SEC_LOCK(sc, descriptors); 412 sc->sc_shutdown = 1; 413 SEC_UNLOCK(sc, descriptors); 414 415 /* Wait until all queued processing finishes */ 416 while (1) { 417 SEC_LOCK(sc, descriptors); 418 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 419 SEC_UNLOCK(sc, descriptors); 420 421 if (i == 0) 422 break; 423 424 if (timeout < 0) { 425 device_printf(dev, "queue flush timeout!\n"); 426 427 /* DMA can be still active - stop it */ 428 for (i = 0; i < SEC_CHANNELS; i++) 429 sec_channel_reset(sc, i, 1); 430 431 break; 432 } 433 434 timeout -= 1000; 435 DELAY(1000); 436 } 437 438 /* Disable interrupts */ 439 SEC_WRITE(sc, SEC_IER, 0); 440 441 /* Unregister from OCF */ 442 crypto_unregister_all(sc->sc_cid); 443 444 /* Free DMA memory */ 445 for (i = 0; i < SEC_DESCRIPTORS; i++) 446 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 447 448 sec_free_dma_mem(&(sc->sc_lt_dmem)); 449 sec_free_dma_mem(&(sc->sc_desc_dmem)); 450 451 /* Release interrupts */ 452 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 453 sc->sc_pri_irid, "primary"); 454 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 455 sc->sc_sec_irid, "secondary"); 456 457 /* Release memory */ 458 if (sc->sc_rres) { 459 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 460 sc->sc_rres); 461 if (error) 462 device_printf(dev, "bus_release_resource() failed for" 463 " I/O memory, error %d\n", error); 464 465 sc->sc_rres = NULL; 466 } 467 468 mtx_destroy(&sc->sc_controller_lock); 469 mtx_destroy(&sc->sc_descriptors_lock); 470 471 return (0); 472 } 473 474 static int 475 sec_suspend(device_t dev) 476 { 477 478 return (0); 479 } 480 481 static int 482 sec_resume(device_t dev) 483 { 484 485 return (0); 486 } 487 488 static int 489 sec_shutdown(device_t dev) 490 { 491 492 return (0); 493 } 494 495 static int 496 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 497 int *irid, driver_intr_t handler, const char *iname) 498 { 499 int error; 500 501 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 502 RF_ACTIVE); 503 504 if ((*ires) == NULL) { 505 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 506 return (ENXIO); 507 } 508 509 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 510 NULL, handler, sc, ihand); 511 512 if (error) { 513 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 514 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 515 device_printf(sc->sc_dev, "could not release %s IRQ\n", 516 iname); 517 518 (*ires) = NULL; 519 return (error); 520 } 521 522 return (0); 523 } 524 525 static void 526 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 527 int irid, const char *iname) 528 { 529 int error; 530 531 if (ires == NULL) 532 return; 533 534 error = bus_teardown_intr(sc->sc_dev, ires, ihand); 535 if (error) 536 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 537 " IRQ, error %d\n", iname, error); 538 539 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 540 if (error) 541 device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 542 " IRQ, error %d\n", iname, error); 543 } 544 545 static void 546 sec_primary_intr(void *arg) 547 { 548 struct sec_softc *sc = arg; 549 struct sec_desc *desc; 550 uint64_t isr; 551 int i, wakeup = 0; 552 553 SEC_LOCK(sc, controller); 554 555 /* Check for errors */ 556 isr = SEC_READ(sc, SEC_ISR); 557 if (isr & sc->sc_int_error_mask) { 558 /* Check each channel for error */ 559 for (i = 0; i < SEC_CHANNELS; i++) { 560 if ((isr & SEC_INT_CH_ERR(i)) == 0) 561 continue; 562 563 device_printf(sc->sc_dev, 564 "I/O error on channel %i!\n", i); 565 566 /* Find and mark problematic descriptor */ 567 desc = sec_find_desc(sc, SEC_READ(sc, 568 SEC_CHAN_CDPR(i))); 569 570 if (desc != NULL) 571 desc->sd_error = EIO; 572 573 /* Do partial channel reset */ 574 sec_channel_reset(sc, i, 0); 575 } 576 } 577 578 /* ACK interrupt */ 579 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 580 581 SEC_UNLOCK(sc, controller); 582 SEC_LOCK(sc, descriptors); 583 584 /* Handle processed descriptors */ 585 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 586 587 while (SEC_QUEUED_DESC_CNT(sc) > 0) { 588 desc = SEC_GET_QUEUED_DESC(sc); 589 590 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 591 SEC_PUT_BACK_QUEUED_DESC(sc); 592 break; 593 } 594 595 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 596 BUS_DMASYNC_PREWRITE); 597 598 desc->sd_crp->crp_etype = desc->sd_error; 599 crypto_done(desc->sd_crp); 600 601 SEC_DESC_FREE_POINTERS(desc); 602 SEC_DESC_FREE_LT(sc, desc); 603 SEC_DESC_QUEUED2FREE(sc); 604 } 605 606 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 607 608 if (!sc->sc_shutdown) { 609 wakeup = sc->sc_blocked; 610 sc->sc_blocked = 0; 611 } 612 613 SEC_UNLOCK(sc, descriptors); 614 615 /* Enqueue ready descriptors in hardware */ 616 sec_enqueue(sc); 617 618 if (wakeup) 619 crypto_unblock(sc->sc_cid, wakeup); 620 } 621 622 static void 623 sec_secondary_intr(void *arg) 624 { 625 struct sec_softc *sc = arg; 626 627 device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 628 sec_primary_intr(arg); 629 } 630 631 static int 632 sec_controller_reset(struct sec_softc *sc) 633 { 634 int timeout = SEC_TIMEOUT; 635 636 /* Reset Controller */ 637 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 638 639 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 640 DELAY(1000); 641 timeout -= 1000; 642 643 if (timeout < 0) { 644 device_printf(sc->sc_dev, "timeout while waiting for " 645 "device reset!\n"); 646 return (ETIMEDOUT); 647 } 648 } 649 650 return (0); 651 } 652 653 static int 654 sec_channel_reset(struct sec_softc *sc, int channel, int full) 655 { 656 int timeout = SEC_TIMEOUT; 657 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 658 uint64_t reg; 659 660 /* Reset Channel */ 661 reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 662 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 663 664 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 665 DELAY(1000); 666 timeout -= 1000; 667 668 if (timeout < 0) { 669 device_printf(sc->sc_dev, "timeout while waiting for " 670 "channel reset!\n"); 671 return (ETIMEDOUT); 672 } 673 } 674 675 if (full) { 676 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 677 678 switch(sc->sc_version) { 679 case 2: 680 reg |= SEC_CHAN_CCR_CDWE; 681 break; 682 case 3: 683 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 684 break; 685 } 686 687 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 688 } 689 690 return (0); 691 } 692 693 static int 694 sec_init(struct sec_softc *sc) 695 { 696 uint64_t reg; 697 int error, i; 698 699 /* Reset controller twice to clear all pending interrupts */ 700 error = sec_controller_reset(sc); 701 if (error) 702 return (error); 703 704 error = sec_controller_reset(sc); 705 if (error) 706 return (error); 707 708 /* Reset channels */ 709 for (i = 0; i < SEC_CHANNELS; i++) { 710 error = sec_channel_reset(sc, i, 1); 711 if (error) 712 return (error); 713 } 714 715 /* Enable Interrupts */ 716 reg = SEC_INT_ITO; 717 for (i = 0; i < SEC_CHANNELS; i++) 718 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 719 720 SEC_WRITE(sc, SEC_IER, reg); 721 722 return (error); 723 } 724 725 static void 726 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 727 { 728 struct sec_dma_mem *dma_mem = arg; 729 730 if (error) 731 return; 732 733 KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 734 dma_mem->dma_paddr = segs->ds_addr; 735 } 736 737 static void 738 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 739 int error) 740 { 741 struct sec_desc_map_info *sdmi = arg; 742 struct sec_softc *sc = sdmi->sdmi_sc; 743 struct sec_lt *lt = NULL; 744 bus_addr_t addr; 745 bus_size_t size; 746 int i; 747 748 SEC_LOCK_ASSERT(sc, descriptors); 749 750 if (error) 751 return; 752 753 for (i = 0; i < nseg; i++) { 754 addr = segs[i].ds_addr; 755 size = segs[i].ds_len; 756 757 /* Skip requested offset */ 758 if (sdmi->sdmi_offset >= size) { 759 sdmi->sdmi_offset -= size; 760 continue; 761 } 762 763 addr += sdmi->sdmi_offset; 764 size -= sdmi->sdmi_offset; 765 sdmi->sdmi_offset = 0; 766 767 /* Do not link more than requested */ 768 if (sdmi->sdmi_size < size) 769 size = sdmi->sdmi_size; 770 771 lt = SEC_ALLOC_LT_ENTRY(sc); 772 lt->sl_lt->shl_length = size; 773 lt->sl_lt->shl_r = 0; 774 lt->sl_lt->shl_n = 0; 775 lt->sl_lt->shl_ptr = addr; 776 777 if (sdmi->sdmi_lt_first == NULL) 778 sdmi->sdmi_lt_first = lt; 779 780 sdmi->sdmi_lt_used += 1; 781 782 if ((sdmi->sdmi_size -= size) == 0) 783 break; 784 } 785 786 sdmi->sdmi_lt_last = lt; 787 } 788 789 static void 790 sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 791 bus_size_t size, int error) 792 { 793 794 sec_dma_map_desc_cb(arg, segs, nseg, error); 795 } 796 797 static int 798 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 799 bus_size_t size) 800 { 801 int error; 802 803 if (dma_mem->dma_vaddr != NULL) 804 return (EBUSY); 805 806 error = bus_dma_tag_create(NULL, /* parent */ 807 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 808 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 809 BUS_SPACE_MAXADDR, /* highaddr */ 810 NULL, NULL, /* filtfunc, filtfuncarg */ 811 size, 1, /* maxsize, nsegments */ 812 size, 0, /* maxsegsz, flags */ 813 NULL, NULL, /* lockfunc, lockfuncarg */ 814 &(dma_mem->dma_tag)); /* dmat */ 815 816 if (error) { 817 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 818 " %i!\n", error); 819 goto err1; 820 } 821 822 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 823 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 824 825 if (error) { 826 device_printf(sc->sc_dev, "failed to allocate DMA safe" 827 " memory, error %i!\n", error); 828 goto err2; 829 } 830 831 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 832 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 833 BUS_DMA_NOWAIT); 834 835 if (error) { 836 device_printf(sc->sc_dev, "cannot get address of the DMA" 837 " memory, error %i\n", error); 838 goto err3; 839 } 840 841 dma_mem->dma_is_map = 0; 842 return (0); 843 844 err3: 845 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 846 err2: 847 bus_dma_tag_destroy(dma_mem->dma_tag); 848 err1: 849 dma_mem->dma_vaddr = NULL; 850 return(error); 851 } 852 853 static int 854 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, 855 bus_size_t size, int type, struct sec_desc_map_info *sdmi) 856 { 857 int error; 858 859 if (dma_mem->dma_vaddr != NULL) 860 return (EBUSY); 861 862 switch (type) { 863 case SEC_MEMORY: 864 break; 865 case SEC_UIO: 866 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 867 break; 868 case SEC_MBUF: 869 size = m_length((struct mbuf*)mem, NULL); 870 break; 871 default: 872 return (EINVAL); 873 } 874 875 error = bus_dma_tag_create(NULL, /* parent */ 876 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 877 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 878 BUS_SPACE_MAXADDR, /* highaddr */ 879 NULL, NULL, /* filtfunc, filtfuncarg */ 880 size, /* maxsize */ 881 SEC_FREE_LT_CNT(sc), /* nsegments */ 882 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 883 NULL, NULL, /* lockfunc, lockfuncarg */ 884 &(dma_mem->dma_tag)); /* dmat */ 885 886 if (error) { 887 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 888 " %i!\n", error); 889 dma_mem->dma_vaddr = NULL; 890 return (error); 891 } 892 893 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 894 895 if (error) { 896 device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 897 "\n", error); 898 bus_dma_tag_destroy(dma_mem->dma_tag); 899 return (error); 900 } 901 902 switch (type) { 903 case SEC_MEMORY: 904 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 905 mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 906 break; 907 case SEC_UIO: 908 error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, 909 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 910 break; 911 case SEC_MBUF: 912 error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, 913 mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 914 break; 915 } 916 917 if (error) { 918 device_printf(sc->sc_dev, "cannot get address of the DMA" 919 " memory, error %i!\n", error); 920 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 921 bus_dma_tag_destroy(dma_mem->dma_tag); 922 return (error); 923 } 924 925 dma_mem->dma_is_map = 1; 926 dma_mem->dma_vaddr = mem; 927 928 return (0); 929 } 930 931 static void 932 sec_free_dma_mem(struct sec_dma_mem *dma_mem) 933 { 934 935 /* Check for double free */ 936 if (dma_mem->dma_vaddr == NULL) 937 return; 938 939 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 940 941 if (dma_mem->dma_is_map) 942 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 943 else 944 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 945 dma_mem->dma_map); 946 947 bus_dma_tag_destroy(dma_mem->dma_tag); 948 dma_mem->dma_vaddr = NULL; 949 } 950 951 static int 952 sec_eu_channel(struct sec_softc *sc, int eu) 953 { 954 uint64_t reg; 955 int channel = 0; 956 957 SEC_LOCK_ASSERT(sc, controller); 958 959 reg = SEC_READ(sc, SEC_EUASR); 960 961 switch (eu) { 962 case SEC_EU_AFEU: 963 channel = SEC_EUASR_AFEU(reg); 964 break; 965 case SEC_EU_DEU: 966 channel = SEC_EUASR_DEU(reg); 967 break; 968 case SEC_EU_MDEU_A: 969 case SEC_EU_MDEU_B: 970 channel = SEC_EUASR_MDEU(reg); 971 break; 972 case SEC_EU_RNGU: 973 channel = SEC_EUASR_RNGU(reg); 974 break; 975 case SEC_EU_PKEU: 976 channel = SEC_EUASR_PKEU(reg); 977 break; 978 case SEC_EU_AESU: 979 channel = SEC_EUASR_AESU(reg); 980 break; 981 case SEC_EU_KEU: 982 channel = SEC_EUASR_KEU(reg); 983 break; 984 case SEC_EU_CRCU: 985 channel = SEC_EUASR_CRCU(reg); 986 break; 987 } 988 989 return (channel - 1); 990 } 991 992 static int 993 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 994 { 995 u_int fflvl = SEC_MAX_FIFO_LEVEL; 996 uint64_t reg; 997 int i; 998 999 SEC_LOCK_ASSERT(sc, controller); 1000 1001 /* Find free channel if have not got one */ 1002 if (channel < 0) { 1003 for (i = 0; i < SEC_CHANNELS; i++) { 1004 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1005 1006 if ((reg & sc->sc_channel_idle_mask) == 0) { 1007 channel = i; 1008 break; 1009 } 1010 } 1011 } 1012 1013 /* There is no free channel */ 1014 if (channel < 0) 1015 return (-1); 1016 1017 /* Check FIFO level on selected channel */ 1018 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 1019 1020 switch(sc->sc_version) { 1021 case 2: 1022 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 1023 break; 1024 case 3: 1025 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 1026 break; 1027 } 1028 1029 if (fflvl >= SEC_MAX_FIFO_LEVEL) 1030 return (-1); 1031 1032 /* Enqueue descriptor in channel */ 1033 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1034 1035 return (channel); 1036 } 1037 1038 static void 1039 sec_enqueue(struct sec_softc *sc) 1040 { 1041 struct sec_desc *desc; 1042 int ch0, ch1; 1043 1044 SEC_LOCK(sc, descriptors); 1045 SEC_LOCK(sc, controller); 1046 1047 while (SEC_READY_DESC_CNT(sc) > 0) { 1048 desc = SEC_GET_READY_DESC(sc); 1049 1050 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1051 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1052 1053 /* 1054 * Both EU are used by the same channel. 1055 * Enqueue descriptor in channel used by busy EUs. 1056 */ 1057 if (ch0 >= 0 && ch0 == ch1) { 1058 if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1059 SEC_DESC_READY2QUEUED(sc); 1060 continue; 1061 } 1062 } 1063 1064 /* 1065 * Only one EU is free. 1066 * Enqueue descriptor in channel used by busy EU. 1067 */ 1068 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1069 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1070 >= 0) { 1071 SEC_DESC_READY2QUEUED(sc); 1072 continue; 1073 } 1074 } 1075 1076 /* 1077 * Both EU are free. 1078 * Enqueue descriptor in first free channel. 1079 */ 1080 if (ch0 < 0 && ch1 < 0) { 1081 if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1082 SEC_DESC_READY2QUEUED(sc); 1083 continue; 1084 } 1085 } 1086 1087 /* Current descriptor can not be queued at the moment */ 1088 SEC_PUT_BACK_READY_DESC(sc); 1089 break; 1090 } 1091 1092 SEC_UNLOCK(sc, controller); 1093 SEC_UNLOCK(sc, descriptors); 1094 } 1095 1096 static struct sec_desc * 1097 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1098 { 1099 struct sec_desc *desc = NULL; 1100 int i; 1101 1102 SEC_LOCK_ASSERT(sc, descriptors); 1103 1104 for (i = 0; i < SEC_CHANNELS; i++) { 1105 if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1106 desc = &(sc->sc_desc[i]); 1107 break; 1108 } 1109 } 1110 1111 return (desc); 1112 } 1113 1114 static int 1115 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1116 bus_addr_t data, bus_size_t dsize) 1117 { 1118 struct sec_hw_desc_ptr *ptr; 1119 1120 SEC_LOCK_ASSERT(sc, descriptors); 1121 1122 ptr = &(desc->sd_desc->shd_pointer[n]); 1123 ptr->shdp_length = dsize; 1124 ptr->shdp_extent = 0; 1125 ptr->shdp_j = 0; 1126 ptr->shdp_ptr = data; 1127 1128 return (0); 1129 } 1130 1131 static int 1132 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1133 u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) 1134 { 1135 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1136 struct sec_hw_desc_ptr *ptr; 1137 int error; 1138 1139 SEC_LOCK_ASSERT(sc, descriptors); 1140 1141 /* For flat memory map only requested region */ 1142 if (dtype == SEC_MEMORY) { 1143 data = (uint8_t*)(data) + doffset; 1144 sdmi.sdmi_offset = 0; 1145 } 1146 1147 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, 1148 dtype, &sdmi); 1149 1150 if (error) 1151 return (error); 1152 1153 sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1154 desc->sd_lt_used += sdmi.sdmi_lt_used; 1155 1156 ptr = &(desc->sd_desc->shd_pointer[n]); 1157 ptr->shdp_length = dsize; 1158 ptr->shdp_extent = 0; 1159 ptr->shdp_j = 1; 1160 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1161 1162 return (0); 1163 } 1164 1165 static int 1166 sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 1167 struct cryptoini **mac) 1168 { 1169 struct cryptoini *e, *m; 1170 1171 e = cri; 1172 m = cri->cri_next; 1173 1174 /* We can haldle only two operations */ 1175 if (m && m->cri_next) 1176 return (EINVAL); 1177 1178 if (sec_mdeu_can_handle(e->cri_alg)) { 1179 cri = m; 1180 m = e; 1181 e = cri; 1182 } 1183 1184 if (m && !sec_mdeu_can_handle(m->cri_alg)) 1185 return (EINVAL); 1186 1187 *enc = e; 1188 *mac = m; 1189 1190 return (0); 1191 } 1192 1193 static int 1194 sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 1195 struct cryptodesc **mac) 1196 { 1197 struct cryptodesc *e, *m, *t; 1198 1199 e = crp->crp_desc; 1200 m = e->crd_next; 1201 1202 /* We can haldle only two operations */ 1203 if (m && m->crd_next) 1204 return (EINVAL); 1205 1206 if (sec_mdeu_can_handle(e->crd_alg)) { 1207 t = m; 1208 m = e; 1209 e = t; 1210 } 1211 1212 if (m && !sec_mdeu_can_handle(m->crd_alg)) 1213 return (EINVAL); 1214 1215 *enc = e; 1216 *mac = m; 1217 1218 return (0); 1219 } 1220 1221 static int 1222 sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) 1223 { 1224 struct sec_softc *sc = device_get_softc(dev); 1225 struct sec_eu_methods *eu = sec_eus; 1226 struct cryptoini *enc = NULL; 1227 struct cryptoini *mac = NULL; 1228 struct sec_session *ses; 1229 int error = -1; 1230 1231 error = sec_split_cri(cri, &enc, &mac); 1232 if (error) 1233 return (error); 1234 1235 /* Check key lengths */ 1236 if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) 1237 return (E2BIG); 1238 1239 if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) 1240 return (E2BIG); 1241 1242 /* Only SEC 3.0 supports digests larger than 256 bits */ 1243 if (sc->sc_version < 3 && mac && mac->cri_klen > 256) 1244 return (E2BIG); 1245 1246 ses = crypto_get_driver_session(cses); 1247 1248 /* Find EU for this session */ 1249 while (eu->sem_make_desc != NULL) { 1250 error = eu->sem_newsession(sc, ses, enc, mac); 1251 if (error >= 0) 1252 break; 1253 1254 eu++; 1255 } 1256 1257 /* If not found, return EINVAL */ 1258 if (error < 0) 1259 return (EINVAL); 1260 1261 /* Save cipher key */ 1262 if (enc && enc->cri_key) { 1263 ses->ss_klen = enc->cri_klen / 8; 1264 memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); 1265 } 1266 1267 /* Save digest key */ 1268 if (mac && mac->cri_key) { 1269 ses->ss_mklen = mac->cri_klen / 8; 1270 memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); 1271 } 1272 1273 ses->ss_eu = eu; 1274 return (0); 1275 } 1276 1277 static int 1278 sec_process(device_t dev, struct cryptop *crp, int hint) 1279 { 1280 struct sec_softc *sc = device_get_softc(dev); 1281 struct sec_desc *desc = NULL; 1282 struct cryptodesc *mac, *enc; 1283 struct sec_session *ses; 1284 int buftype, error = 0; 1285 1286 ses = crypto_get_driver_session(crp->crp_session); 1287 1288 /* Check for input length */ 1289 if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { 1290 crp->crp_etype = E2BIG; 1291 crypto_done(crp); 1292 return (0); 1293 } 1294 1295 /* Get descriptors */ 1296 if (sec_split_crp(crp, &enc, &mac)) { 1297 crp->crp_etype = EINVAL; 1298 crypto_done(crp); 1299 return (0); 1300 } 1301 1302 SEC_LOCK(sc, descriptors); 1303 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1304 1305 /* Block driver if there is no free descriptors or we are going down */ 1306 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1307 sc->sc_blocked |= CRYPTO_SYMQ; 1308 SEC_UNLOCK(sc, descriptors); 1309 return (ERESTART); 1310 } 1311 1312 /* Prepare descriptor */ 1313 desc = SEC_GET_FREE_DESC(sc); 1314 desc->sd_lt_used = 0; 1315 desc->sd_error = 0; 1316 desc->sd_crp = crp; 1317 1318 if (crp->crp_flags & CRYPTO_F_IOV) 1319 buftype = SEC_UIO; 1320 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1321 buftype = SEC_MBUF; 1322 else 1323 buftype = SEC_MEMORY; 1324 1325 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1326 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1327 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1328 ses->ss_ivlen); 1329 else 1330 arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); 1331 1332 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1333 crypto_copyback(crp->crp_flags, crp->crp_buf, 1334 enc->crd_inject, ses->ss_ivlen, 1335 desc->sd_desc->shd_iv); 1336 } else if (enc) { 1337 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1338 memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 1339 ses->ss_ivlen); 1340 else 1341 crypto_copydata(crp->crp_flags, crp->crp_buf, 1342 enc->crd_inject, ses->ss_ivlen, 1343 desc->sd_desc->shd_iv); 1344 } 1345 1346 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1347 if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1348 ses->ss_klen = enc->crd_klen / 8; 1349 memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); 1350 } else 1351 error = E2BIG; 1352 } 1353 1354 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1355 if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 1356 ses->ss_mklen = mac->crd_klen / 8; 1357 memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); 1358 } else 1359 error = E2BIG; 1360 } 1361 1362 if (!error) { 1363 memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); 1364 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); 1365 1366 error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); 1367 } 1368 1369 if (error) { 1370 SEC_DESC_FREE_POINTERS(desc); 1371 SEC_DESC_PUT_BACK_LT(sc, desc); 1372 SEC_PUT_BACK_FREE_DESC(sc); 1373 SEC_UNLOCK(sc, descriptors); 1374 crp->crp_etype = error; 1375 crypto_done(crp); 1376 return (0); 1377 } 1378 1379 /* 1380 * Skip DONE interrupt if this is not last request in burst, but only 1381 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1382 * signaling on each descriptor. 1383 */ 1384 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1385 desc->sd_desc->shd_dn = 0; 1386 else 1387 desc->sd_desc->shd_dn = 1; 1388 1389 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1390 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1391 BUS_DMASYNC_POSTWRITE); 1392 SEC_DESC_FREE2READY(sc); 1393 SEC_UNLOCK(sc, descriptors); 1394 1395 /* Enqueue ready descriptors in hardware */ 1396 sec_enqueue(sc); 1397 1398 return (0); 1399 } 1400 1401 static int 1402 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1403 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1404 int buftype) 1405 { 1406 struct sec_hw_desc *hd = desc->sd_desc; 1407 int error; 1408 1409 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1410 hd->shd_eu_sel1 = SEC_EU_NONE; 1411 hd->shd_mode1 = 0; 1412 1413 /* Pointer 0: NULL */ 1414 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1415 if (error) 1416 return (error); 1417 1418 /* Pointer 1: IV IN */ 1419 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1420 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1421 if (error) 1422 return (error); 1423 1424 /* Pointer 2: Cipher Key */ 1425 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1426 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1427 if (error) 1428 return (error); 1429 1430 /* Pointer 3: Data IN */ 1431 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, 1432 enc->crd_len, buftype); 1433 if (error) 1434 return (error); 1435 1436 /* Pointer 4: Data OUT */ 1437 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1438 enc->crd_len, buftype); 1439 if (error) 1440 return (error); 1441 1442 /* Pointer 5: IV OUT (Not used: NULL) */ 1443 error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1444 if (error) 1445 return (error); 1446 1447 /* Pointer 6: NULL */ 1448 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1449 1450 return (error); 1451 } 1452 1453 static int 1454 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1455 struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 1456 struct cryptodesc *mac, int buftype) 1457 { 1458 struct sec_hw_desc *hd = desc->sd_desc; 1459 u_int eu, mode, hashlen; 1460 int error; 1461 1462 if (mac->crd_len < enc->crd_len) 1463 return (EINVAL); 1464 1465 if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) 1466 return (EINVAL); 1467 1468 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1469 if (error) 1470 return (error); 1471 1472 hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1473 hd->shd_eu_sel1 = eu; 1474 hd->shd_mode1 = mode; 1475 1476 /* Pointer 0: HMAC Key */ 1477 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1478 offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); 1479 if (error) 1480 return (error); 1481 1482 /* Pointer 1: HMAC-Only Data IN */ 1483 error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, 1484 mac->crd_len - enc->crd_len, buftype); 1485 if (error) 1486 return (error); 1487 1488 /* Pointer 2: Cipher Key */ 1489 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1490 offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 1491 if (error) 1492 return (error); 1493 1494 /* Pointer 3: IV IN */ 1495 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1496 offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 1497 if (error) 1498 return (error); 1499 1500 /* Pointer 4: Data IN */ 1501 error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 1502 enc->crd_len, buftype); 1503 if (error) 1504 return (error); 1505 1506 /* Pointer 5: Data OUT */ 1507 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, 1508 enc->crd_len, buftype); 1509 if (error) 1510 return (error); 1511 1512 /* Pointer 6: HMAC OUT */ 1513 error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, 1514 hashlen, buftype); 1515 1516 return (error); 1517 } 1518 1519 /* AESU */ 1520 1521 static int 1522 sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, 1523 struct cryptoini *enc, struct cryptoini *mac) 1524 { 1525 1526 if (enc == NULL) 1527 return (-1); 1528 1529 if (enc->cri_alg != CRYPTO_AES_CBC) 1530 return (-1); 1531 1532 ses->ss_ivlen = AES_BLOCK_LEN; 1533 1534 return (0); 1535 } 1536 1537 static int 1538 sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1539 struct sec_desc *desc, struct cryptop *crp, int buftype) 1540 { 1541 struct sec_hw_desc *hd = desc->sd_desc; 1542 struct cryptodesc *enc, *mac; 1543 int error; 1544 1545 error = sec_split_crp(crp, &enc, &mac); 1546 if (error) 1547 return (error); 1548 1549 if (!enc) 1550 return (EINVAL); 1551 1552 hd->shd_eu_sel0 = SEC_EU_AESU; 1553 hd->shd_mode0 = SEC_AESU_MODE_CBC; 1554 1555 if (enc->crd_alg != CRYPTO_AES_CBC) 1556 return (EINVAL); 1557 1558 if (enc->crd_flags & CRD_F_ENCRYPT) { 1559 hd->shd_mode0 |= SEC_AESU_MODE_ED; 1560 hd->shd_dir = 0; 1561 } else 1562 hd->shd_dir = 1; 1563 1564 if (mac) 1565 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1566 buftype); 1567 else 1568 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1569 buftype); 1570 1571 return (error); 1572 } 1573 1574 /* DEU */ 1575 1576 static int 1577 sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, 1578 struct cryptoini *enc, struct cryptoini *mac) 1579 { 1580 1581 if (enc == NULL) 1582 return (-1); 1583 1584 switch (enc->cri_alg) { 1585 case CRYPTO_DES_CBC: 1586 case CRYPTO_3DES_CBC: 1587 break; 1588 default: 1589 return (-1); 1590 } 1591 1592 ses->ss_ivlen = DES_BLOCK_LEN; 1593 1594 return (0); 1595 } 1596 1597 static int 1598 sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1599 struct sec_desc *desc, struct cryptop *crp, int buftype) 1600 { 1601 struct sec_hw_desc *hd = desc->sd_desc; 1602 struct cryptodesc *enc, *mac; 1603 int error; 1604 1605 error = sec_split_crp(crp, &enc, &mac); 1606 if (error) 1607 return (error); 1608 1609 if (!enc) 1610 return (EINVAL); 1611 1612 hd->shd_eu_sel0 = SEC_EU_DEU; 1613 hd->shd_mode0 = SEC_DEU_MODE_CBC; 1614 1615 switch (enc->crd_alg) { 1616 case CRYPTO_3DES_CBC: 1617 hd->shd_mode0 |= SEC_DEU_MODE_TS; 1618 break; 1619 case CRYPTO_DES_CBC: 1620 break; 1621 default: 1622 return (EINVAL); 1623 } 1624 1625 if (enc->crd_flags & CRD_F_ENCRYPT) { 1626 hd->shd_mode0 |= SEC_DEU_MODE_ED; 1627 hd->shd_dir = 0; 1628 } else 1629 hd->shd_dir = 1; 1630 1631 if (mac) 1632 error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 1633 buftype); 1634 else 1635 error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 1636 buftype); 1637 1638 return (error); 1639 } 1640 1641 /* MDEU */ 1642 1643 static int 1644 sec_mdeu_can_handle(u_int alg) 1645 { 1646 switch (alg) { 1647 case CRYPTO_MD5: 1648 case CRYPTO_SHA1: 1649 case CRYPTO_MD5_HMAC: 1650 case CRYPTO_SHA1_HMAC: 1651 case CRYPTO_SHA2_256_HMAC: 1652 case CRYPTO_SHA2_384_HMAC: 1653 case CRYPTO_SHA2_512_HMAC: 1654 return (1); 1655 default: 1656 return (0); 1657 } 1658 } 1659 1660 static int 1661 sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) 1662 { 1663 1664 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1665 *eu = SEC_EU_NONE; 1666 1667 switch (crd->crd_alg) { 1668 case CRYPTO_MD5_HMAC: 1669 *mode |= SEC_MDEU_MODE_HMAC; 1670 /* FALLTHROUGH */ 1671 case CRYPTO_MD5: 1672 *eu = SEC_EU_MDEU_A; 1673 *mode |= SEC_MDEU_MODE_MD5; 1674 *hashlen = MD5_HASH_LEN; 1675 break; 1676 case CRYPTO_SHA1_HMAC: 1677 *mode |= SEC_MDEU_MODE_HMAC; 1678 /* FALLTHROUGH */ 1679 case CRYPTO_SHA1: 1680 *eu = SEC_EU_MDEU_A; 1681 *mode |= SEC_MDEU_MODE_SHA1; 1682 *hashlen = SHA1_HASH_LEN; 1683 break; 1684 case CRYPTO_SHA2_256_HMAC: 1685 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1686 *eu = SEC_EU_MDEU_A; 1687 break; 1688 case CRYPTO_SHA2_384_HMAC: 1689 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1690 *eu = SEC_EU_MDEU_B; 1691 break; 1692 case CRYPTO_SHA2_512_HMAC: 1693 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1694 *eu = SEC_EU_MDEU_B; 1695 break; 1696 default: 1697 return (EINVAL); 1698 } 1699 1700 if (*mode & SEC_MDEU_MODE_HMAC) 1701 *hashlen = SEC_HMAC_HASH_LEN; 1702 1703 return (0); 1704 } 1705 1706 static int 1707 sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, 1708 struct cryptoini *enc, struct cryptoini *mac) 1709 { 1710 1711 if (mac && sec_mdeu_can_handle(mac->cri_alg)) 1712 return (0); 1713 1714 return (-1); 1715 } 1716 1717 static int 1718 sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, 1719 struct sec_desc *desc, struct cryptop *crp, int buftype) 1720 { 1721 struct cryptodesc *enc, *mac; 1722 struct sec_hw_desc *hd = desc->sd_desc; 1723 u_int eu, mode, hashlen; 1724 int error; 1725 1726 error = sec_split_crp(crp, &enc, &mac); 1727 if (error) 1728 return (error); 1729 1730 if (enc) 1731 return (EINVAL); 1732 1733 error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 1734 if (error) 1735 return (error); 1736 1737 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1738 hd->shd_eu_sel0 = eu; 1739 hd->shd_mode0 = mode; 1740 hd->shd_eu_sel1 = SEC_EU_NONE; 1741 hd->shd_mode1 = 0; 1742 1743 /* Pointer 0: NULL */ 1744 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1745 if (error) 1746 return (error); 1747 1748 /* Pointer 1: Context In (Not used: NULL) */ 1749 error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1750 if (error) 1751 return (error); 1752 1753 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1754 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1755 error = sec_make_pointer_direct(sc, desc, 2, 1756 desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1757 shd_mkey), ses->ss_mklen); 1758 else 1759 error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1760 1761 if (error) 1762 return (error); 1763 1764 /* Pointer 3: Input Data */ 1765 error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, 1766 mac->crd_len, buftype); 1767 if (error) 1768 return (error); 1769 1770 /* Pointer 4: NULL */ 1771 error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1772 if (error) 1773 return (error); 1774 1775 /* Pointer 5: Hash out */ 1776 error = sec_make_pointer(sc, desc, 5, crp->crp_buf, 1777 mac->crd_inject, hashlen, buftype); 1778 if (error) 1779 return (error); 1780 1781 /* Pointer 6: NULL */ 1782 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1783 1784 return (0); 1785 } 1786