1 /*- 2 * Copyright (C) 2009-2011 Semihalf. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * CESA SRAM Memory Map: 29 * 30 * +------------------------+ <= sc->sc_sram_base + CESA_SRAM_SIZE 31 * | | 32 * | DATA | 33 * | | 34 * +------------------------+ <= sc->sc_sram_base + CESA_DATA(0) 35 * | struct cesa_sa_data | 36 * +------------------------+ 37 * | struct cesa_sa_hdesc | 38 * +------------------------+ <= sc->sc_sram_base 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/kernel.h> 49 #include <sys/lock.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/mutex.h> 53 #include <sys/rman.h> 54 55 #include <machine/bus.h> 56 #include <machine/intr.h> 57 #include <machine/resource.h> 58 59 #include <dev/fdt/fdt_common.h> 60 #include <dev/ofw/ofw_bus.h> 61 #include <dev/ofw/ofw_bus_subr.h> 62 63 #include <sys/md5.h> 64 #include <crypto/sha1.h> 65 #include <crypto/rijndael/rijndael.h> 66 #include <opencrypto/cryptodev.h> 67 #include "cryptodev_if.h" 68 69 #include <arm/mv/mvreg.h> 70 #include <arm/mv/mvwin.h> 71 #include <arm/mv/mvvar.h> 72 #include "cesa.h" 73 74 #undef DEBUG 75 76 static int cesa_probe(device_t); 77 static int cesa_attach(device_t); 78 static int cesa_detach(device_t); 79 static void cesa_intr(void *); 80 static int cesa_newsession(device_t, u_int32_t *, struct cryptoini *); 81 static int cesa_freesession(device_t, u_int64_t); 82 static int cesa_process(device_t, struct cryptop *, int); 83 84 static struct resource_spec cesa_res_spec[] = { 85 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 86 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 87 { -1, 0 } 88 }; 89 90 static device_method_t cesa_methods[] = { 91 /* Device interface */ 92 DEVMETHOD(device_probe, cesa_probe), 93 DEVMETHOD(device_attach, cesa_attach), 94 DEVMETHOD(device_detach, cesa_detach), 95 96 /* Bus interface */ 97 DEVMETHOD(bus_print_child, bus_generic_print_child), 98 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 99 100 /* Crypto device methods */ 101 DEVMETHOD(cryptodev_newsession, cesa_newsession), 102 DEVMETHOD(cryptodev_freesession,cesa_freesession), 103 DEVMETHOD(cryptodev_process, cesa_process), 104 105 { 0, 0 } 106 }; 107 108 static driver_t cesa_driver = { 109 "cesa", 110 cesa_methods, 111 sizeof (struct cesa_softc) 112 }; 113 static devclass_t cesa_devclass; 114 115 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); 116 MODULE_DEPEND(cesa, crypto, 1, 1, 1); 117 118 static void 119 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 120 { 121 #ifdef DEBUG 122 device_t dev; 123 124 dev = sc->sc_dev; 125 device_printf(dev, "CESA SA Hardware Descriptor:\n"); 126 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 127 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 128 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 129 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 130 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 131 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 132 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 133 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 134 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 135 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 136 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 137 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 138 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 139 #endif 140 } 141 142 static void 143 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 144 { 145 struct cesa_dma_mem *cdm; 146 147 if (error) 148 return; 149 150 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 151 cdm = arg; 152 cdm->cdm_paddr = segs->ds_addr; 153 } 154 155 static int 156 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 157 bus_size_t size) 158 { 159 int error; 160 161 KASSERT(cdm->cdm_vaddr == NULL, 162 ("%s(): DMA memory descriptor in use.", __func__)); 163 164 error = bus_dma_tag_create(NULL, /* parent */ 165 PAGE_SIZE, 0, /* alignment, boundary */ 166 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 167 BUS_SPACE_MAXADDR, /* highaddr */ 168 NULL, NULL, /* filtfunc, filtfuncarg */ 169 size, 1, /* maxsize, nsegments */ 170 size, 0, /* maxsegsz, flags */ 171 NULL, NULL, /* lockfunc, lockfuncarg */ 172 &cdm->cdm_tag); /* dmat */ 173 if (error) { 174 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 175 " %i!\n", error); 176 177 goto err1; 178 } 179 180 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 181 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 182 if (error) { 183 device_printf(sc->sc_dev, "failed to allocate DMA safe" 184 " memory, error %i!\n", error); 185 186 goto err2; 187 } 188 189 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 190 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 191 if (error) { 192 device_printf(sc->sc_dev, "cannot get address of the DMA" 193 " memory, error %i\n", error); 194 195 goto err3; 196 } 197 198 return (0); 199 err3: 200 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 201 err2: 202 bus_dma_tag_destroy(cdm->cdm_tag); 203 err1: 204 cdm->cdm_vaddr = NULL; 205 return (error); 206 } 207 208 static void 209 cesa_free_dma_mem(struct cesa_dma_mem *cdm) 210 { 211 212 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 213 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 214 bus_dma_tag_destroy(cdm->cdm_tag); 215 cdm->cdm_vaddr = NULL; 216 } 217 218 static void 219 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 220 { 221 222 /* Sync only if dma memory is valid */ 223 if (cdm->cdm_vaddr != NULL) 224 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 225 } 226 227 static void 228 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 229 { 230 231 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 232 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 233 cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 234 } 235 236 static struct cesa_session * 237 cesa_alloc_session(struct cesa_softc *sc) 238 { 239 struct cesa_session *cs; 240 241 CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions); 242 243 return (cs); 244 } 245 246 static struct cesa_session * 247 cesa_get_session(struct cesa_softc *sc, uint32_t sid) 248 { 249 250 if (sid >= CESA_SESSIONS) 251 return (NULL); 252 253 return (&sc->sc_sessions[sid]); 254 } 255 256 static void 257 cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs) 258 { 259 260 CESA_GENERIC_FREE_LOCKED(sc, cs, sessions); 261 } 262 263 static struct cesa_request * 264 cesa_alloc_request(struct cesa_softc *sc) 265 { 266 struct cesa_request *cr; 267 268 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 269 if (!cr) 270 return (NULL); 271 272 STAILQ_INIT(&cr->cr_tdesc); 273 STAILQ_INIT(&cr->cr_sdesc); 274 275 return (cr); 276 } 277 278 static void 279 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 280 { 281 282 /* Free TDMA descriptors assigned to this request */ 283 CESA_LOCK(sc, tdesc); 284 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 285 CESA_UNLOCK(sc, tdesc); 286 287 /* Free SA descriptors assigned to this request */ 288 CESA_LOCK(sc, sdesc); 289 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 290 CESA_UNLOCK(sc, sdesc); 291 292 /* Unload DMA memory asociated with request */ 293 if (cr->cr_dmap_loaded) { 294 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 295 cr->cr_dmap_loaded = 0; 296 } 297 298 CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 299 } 300 301 static void 302 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 303 { 304 305 CESA_LOCK(sc, requests); 306 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 307 CESA_UNLOCK(sc, requests); 308 } 309 310 static struct cesa_tdma_desc * 311 cesa_alloc_tdesc(struct cesa_softc *sc) 312 { 313 struct cesa_tdma_desc *ctd; 314 315 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 316 317 if (!ctd) 318 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 319 "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 320 321 return (ctd); 322 } 323 324 static struct cesa_sa_desc * 325 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 326 { 327 struct cesa_sa_desc *csd; 328 329 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 330 if (!csd) { 331 device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 332 "Consider increasing CESA_SA_DESCRIPTORS.\n"); 333 return (NULL); 334 } 335 336 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 337 338 /* Fill-in SA descriptor with default values */ 339 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 340 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 341 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 342 csd->csd_cshd->cshd_enc_src = 0; 343 csd->csd_cshd->cshd_enc_dst = 0; 344 csd->csd_cshd->cshd_enc_dlen = 0; 345 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 346 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 347 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 348 csd->csd_cshd->cshd_mac_src = 0; 349 csd->csd_cshd->cshd_mac_dlen = 0; 350 351 return (csd); 352 } 353 354 static struct cesa_tdma_desc * 355 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 356 bus_size_t size) 357 { 358 struct cesa_tdma_desc *ctd; 359 360 ctd = cesa_alloc_tdesc(sc); 361 if (!ctd) 362 return (NULL); 363 364 ctd->ctd_cthd->cthd_dst = dst; 365 ctd->ctd_cthd->cthd_src = src; 366 ctd->ctd_cthd->cthd_byte_count = size; 367 368 /* Handle special control packet */ 369 if (size != 0) 370 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 371 else 372 ctd->ctd_cthd->cthd_flags = 0; 373 374 return (ctd); 375 } 376 377 static struct cesa_tdma_desc * 378 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 379 { 380 381 return (cesa_tdma_copy(sc, sc->sc_sram_base + 382 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 383 sizeof(struct cesa_sa_data))); 384 } 385 386 static struct cesa_tdma_desc * 387 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 388 { 389 390 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base + 391 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 392 } 393 394 static struct cesa_tdma_desc * 395 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 396 { 397 398 return (cesa_tdma_copy(sc, sc->sc_sram_base, csd->csd_cshd_paddr, 399 sizeof(struct cesa_sa_hdesc))); 400 } 401 402 static void 403 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 404 { 405 struct cesa_tdma_desc *ctd_prev; 406 407 if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 408 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 409 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 410 } 411 412 ctd->ctd_cthd->cthd_next = 0; 413 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 414 } 415 416 static int 417 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 418 struct cesa_packet *cp, struct cesa_sa_desc *csd) 419 { 420 struct cesa_tdma_desc *ctd, *tmp; 421 422 /* Copy SA descriptor for this packet */ 423 ctd = cesa_tdma_copy_sdesc(sc, csd); 424 if (!ctd) 425 return (ENOMEM); 426 427 cesa_append_tdesc(cr, ctd); 428 429 /* Copy data to be processed */ 430 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 431 cesa_append_tdesc(cr, ctd); 432 STAILQ_INIT(&cp->cp_copyin); 433 434 /* Insert control descriptor */ 435 ctd = cesa_tdma_copy(sc, 0, 0, 0); 436 if (!ctd) 437 return (ENOMEM); 438 439 cesa_append_tdesc(cr, ctd); 440 441 /* Copy back results */ 442 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 443 cesa_append_tdesc(cr, ctd); 444 STAILQ_INIT(&cp->cp_copyout); 445 446 return (0); 447 } 448 449 static int 450 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 451 { 452 uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; 453 uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; 454 SHA1_CTX sha1ctx; 455 MD5_CTX md5ctx; 456 uint32_t *hout; 457 uint32_t *hin; 458 int i; 459 460 memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 461 memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 462 for (i = 0; i < mklen; i++) { 463 ipad[i] ^= mkey[i]; 464 opad[i] ^= mkey[i]; 465 } 466 467 hin = (uint32_t *)cs->cs_hiv_in; 468 hout = (uint32_t *)cs->cs_hiv_out; 469 470 switch (alg) { 471 case CRYPTO_MD5_HMAC: 472 MD5Init(&md5ctx); 473 MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN); 474 memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); 475 MD5Init(&md5ctx); 476 MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN); 477 memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); 478 break; 479 case CRYPTO_SHA1_HMAC: 480 SHA1Init(&sha1ctx); 481 SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN); 482 memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 483 SHA1Init(&sha1ctx); 484 SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN); 485 memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 486 break; 487 default: 488 return (EINVAL); 489 } 490 491 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 492 hin[i] = htobe32(hin[i]); 493 hout[i] = htobe32(hout[i]); 494 } 495 496 return (0); 497 } 498 499 static int 500 cesa_prep_aes_key(struct cesa_session *cs) 501 { 502 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 503 uint32_t *dkey; 504 int i; 505 506 rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); 507 508 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 509 dkey = (uint32_t *)cs->cs_aes_dkey; 510 511 switch (cs->cs_klen) { 512 case 16: 513 cs->cs_config |= CESA_CSH_AES_KLEN_128; 514 for (i = 0; i < 4; i++) 515 *dkey++ = htobe32(ek[4 * 10 + i]); 516 break; 517 case 24: 518 cs->cs_config |= CESA_CSH_AES_KLEN_192; 519 for (i = 0; i < 4; i++) 520 *dkey++ = htobe32(ek[4 * 12 + i]); 521 for (i = 0; i < 2; i++) 522 *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 523 break; 524 case 32: 525 cs->cs_config |= CESA_CSH_AES_KLEN_256; 526 for (i = 0; i < 4; i++) 527 *dkey++ = htobe32(ek[4 * 14 + i]); 528 for (i = 0; i < 4; i++) 529 *dkey++ = htobe32(ek[4 * 13 + i]); 530 break; 531 default: 532 return (EINVAL); 533 } 534 535 return (0); 536 } 537 538 static int 539 cesa_is_hash(int alg) 540 { 541 542 switch (alg) { 543 case CRYPTO_MD5: 544 case CRYPTO_MD5_HMAC: 545 case CRYPTO_SHA1: 546 case CRYPTO_SHA1_HMAC: 547 return (1); 548 default: 549 return (0); 550 } 551 } 552 553 static void 554 cesa_start_packet(struct cesa_packet *cp, unsigned int size) 555 { 556 557 cp->cp_size = size; 558 cp->cp_offset = 0; 559 STAILQ_INIT(&cp->cp_copyin); 560 STAILQ_INIT(&cp->cp_copyout); 561 } 562 563 static int 564 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 565 bus_dma_segment_t *seg) 566 { 567 struct cesa_tdma_desc *ctd; 568 unsigned int bsize; 569 570 /* Calculate size of block copy */ 571 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 572 573 if (bsize > 0) { 574 ctd = cesa_tdma_copy(sc, sc->sc_sram_base + 575 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 576 if (!ctd) 577 return (-ENOMEM); 578 579 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 580 581 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base + 582 CESA_DATA(cp->cp_offset), bsize); 583 if (!ctd) 584 return (-ENOMEM); 585 586 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 587 588 seg->ds_len -= bsize; 589 seg->ds_addr += bsize; 590 cp->cp_offset += bsize; 591 } 592 593 return (bsize); 594 } 595 596 static void 597 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 598 { 599 unsigned int mpsize, fragmented; 600 unsigned int mlen, mskip, tmlen; 601 struct cesa_chain_info *cci; 602 unsigned int elen, eskip; 603 unsigned int skip, len; 604 struct cesa_sa_desc *csd; 605 struct cesa_request *cr; 606 struct cesa_softc *sc; 607 struct cesa_packet cp; 608 bus_dma_segment_t seg; 609 uint32_t config; 610 int size; 611 612 cci = arg; 613 sc = cci->cci_sc; 614 cr = cci->cci_cr; 615 616 if (error) { 617 cci->cci_error = error; 618 return; 619 } 620 621 elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; 622 eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; 623 mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; 624 mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; 625 626 if (elen && mlen && 627 ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || 628 (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || 629 (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { 630 /* 631 * Data alignment in the request does not meet CESA requiremnts 632 * for combined encryption/decryption and hashing. We have to 633 * split the request to separate operations and process them 634 * one by one. 635 */ 636 config = cci->cci_config; 637 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 638 config &= ~CESA_CSHD_OP_MASK; 639 640 cci->cci_config = config | CESA_CSHD_MAC; 641 cci->cci_enc = NULL; 642 cci->cci_mac = cr->cr_mac; 643 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 644 645 cci->cci_config = config | CESA_CSHD_ENC; 646 cci->cci_enc = cr->cr_enc; 647 cci->cci_mac = NULL; 648 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 649 } else { 650 config &= ~CESA_CSHD_OP_MASK; 651 652 cci->cci_config = config | CESA_CSHD_ENC; 653 cci->cci_enc = cr->cr_enc; 654 cci->cci_mac = NULL; 655 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 656 657 cci->cci_config = config | CESA_CSHD_MAC; 658 cci->cci_enc = NULL; 659 cci->cci_mac = cr->cr_mac; 660 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 661 } 662 663 return; 664 } 665 666 tmlen = mlen; 667 fragmented = 0; 668 mpsize = CESA_MAX_PACKET_SIZE; 669 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 670 671 if (elen && mlen) { 672 skip = MIN(eskip, mskip); 673 len = MAX(elen + eskip, mlen + mskip) - skip; 674 } else if (elen) { 675 skip = eskip; 676 len = elen; 677 } else { 678 skip = mskip; 679 len = mlen; 680 } 681 682 /* Start first packet in chain */ 683 cesa_start_packet(&cp, MIN(mpsize, len)); 684 685 while (nseg-- && len > 0) { 686 seg = *(segs++); 687 688 /* 689 * Skip data in buffer on which neither ENC nor MAC operation 690 * is requested. 691 */ 692 if (skip > 0) { 693 size = MIN(skip, seg.ds_len); 694 skip -= size; 695 696 seg.ds_addr += size; 697 seg.ds_len -= size; 698 699 if (eskip > 0) 700 eskip -= size; 701 702 if (mskip > 0) 703 mskip -= size; 704 705 if (seg.ds_len == 0) 706 continue; 707 } 708 709 while (1) { 710 /* 711 * Fill in current packet with data. Break if there is 712 * no more data in current DMA segment or an error 713 * occured. 714 */ 715 size = cesa_fill_packet(sc, &cp, &seg); 716 if (size <= 0) { 717 error = -size; 718 break; 719 } 720 721 len -= size; 722 723 /* If packet is full, append it to the chain */ 724 if (cp.cp_size == cp.cp_offset) { 725 csd = cesa_alloc_sdesc(sc, cr); 726 if (!csd) { 727 error = ENOMEM; 728 break; 729 } 730 731 /* Create SA descriptor for this packet */ 732 csd->csd_cshd->cshd_config = cci->cci_config; 733 csd->csd_cshd->cshd_mac_total_dlen = tmlen; 734 735 /* 736 * Enable fragmentation if request will not fit 737 * into one packet. 738 */ 739 if (len > 0) { 740 if (!fragmented) { 741 fragmented = 1; 742 csd->csd_cshd->cshd_config |= 743 CESA_CSHD_FRAG_FIRST; 744 } else 745 csd->csd_cshd->cshd_config |= 746 CESA_CSHD_FRAG_MIDDLE; 747 } else if (fragmented) 748 csd->csd_cshd->cshd_config |= 749 CESA_CSHD_FRAG_LAST; 750 751 if (eskip < cp.cp_size && elen > 0) { 752 csd->csd_cshd->cshd_enc_src = 753 CESA_DATA(eskip); 754 csd->csd_cshd->cshd_enc_dst = 755 CESA_DATA(eskip); 756 csd->csd_cshd->cshd_enc_dlen = 757 MIN(elen, cp.cp_size - eskip); 758 } 759 760 if (mskip < cp.cp_size && mlen > 0) { 761 csd->csd_cshd->cshd_mac_src = 762 CESA_DATA(mskip); 763 csd->csd_cshd->cshd_mac_dlen = 764 MIN(mlen, cp.cp_size - mskip); 765 } 766 767 elen -= csd->csd_cshd->cshd_enc_dlen; 768 eskip -= MIN(eskip, cp.cp_size); 769 mlen -= csd->csd_cshd->cshd_mac_dlen; 770 mskip -= MIN(mskip, cp.cp_size); 771 772 cesa_dump_cshd(sc, csd->csd_cshd); 773 774 /* Append packet to the request */ 775 error = cesa_append_packet(sc, cr, &cp, csd); 776 if (error) 777 break; 778 779 /* Start a new packet, as current is full */ 780 cesa_start_packet(&cp, MIN(mpsize, len)); 781 } 782 } 783 784 if (error) 785 break; 786 } 787 788 if (error) { 789 /* 790 * Move all allocated resources to the request. They will be 791 * freed later. 792 */ 793 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 794 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 795 cci->cci_error = error; 796 } 797 } 798 799 static void 800 cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 801 bus_size_t size, int error) 802 { 803 804 cesa_create_chain_cb(arg, segs, nseg, error); 805 } 806 807 static int 808 cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) 809 { 810 struct cesa_chain_info cci; 811 struct cesa_tdma_desc *ctd; 812 uint32_t config; 813 int error; 814 815 error = 0; 816 CESA_LOCK_ASSERT(sc, sessions); 817 818 /* Create request metadata */ 819 if (cr->cr_enc) { 820 if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && 821 (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 822 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 823 cr->cr_cs->cs_klen); 824 else 825 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 826 cr->cr_cs->cs_klen); 827 } 828 829 if (cr->cr_mac) { 830 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 831 CESA_MAX_HASH_LEN); 832 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 833 CESA_MAX_HASH_LEN); 834 } 835 836 ctd = cesa_tdma_copyin_sa_data(sc, cr); 837 if (!ctd) 838 return (ENOMEM); 839 840 cesa_append_tdesc(cr, ctd); 841 842 /* Prepare SA configuration */ 843 config = cr->cr_cs->cs_config; 844 845 if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 846 config |= CESA_CSHD_DECRYPT; 847 if (cr->cr_enc && !cr->cr_mac) 848 config |= CESA_CSHD_ENC; 849 if (!cr->cr_enc && cr->cr_mac) 850 config |= CESA_CSHD_MAC; 851 if (cr->cr_enc && cr->cr_mac) 852 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 853 CESA_CSHD_ENC_AND_MAC; 854 855 /* Create data packets */ 856 cci.cci_sc = sc; 857 cci.cci_cr = cr; 858 cci.cci_enc = cr->cr_enc; 859 cci.cci_mac = cr->cr_mac; 860 cci.cci_config = config; 861 cci.cci_error = 0; 862 863 if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) 864 error = bus_dmamap_load_uio(sc->sc_data_dtag, 865 cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, 866 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 867 else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) 868 error = bus_dmamap_load_mbuf(sc->sc_data_dtag, 869 cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, 870 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 871 else 872 error = bus_dmamap_load(sc->sc_data_dtag, 873 cr->cr_dmap, cr->cr_crp->crp_buf, 874 cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, 875 BUS_DMA_NOWAIT); 876 877 if (!error) 878 cr->cr_dmap_loaded = 1; 879 880 if (cci.cci_error) 881 error = cci.cci_error; 882 883 if (error) 884 return (error); 885 886 /* Read back request metadata */ 887 ctd = cesa_tdma_copyout_sa_data(sc, cr); 888 if (!ctd) 889 return (ENOMEM); 890 891 cesa_append_tdesc(cr, ctd); 892 893 return (0); 894 } 895 896 static void 897 cesa_execute(struct cesa_softc *sc) 898 { 899 struct cesa_tdma_desc *prev_ctd, *ctd; 900 struct cesa_request *prev_cr, *cr; 901 902 CESA_LOCK(sc, requests); 903 904 /* 905 * If ready list is empty, there is nothing to execute. If queued list 906 * is not empty, the hardware is busy and we cannot start another 907 * execution. 908 */ 909 if (STAILQ_EMPTY(&sc->sc_ready_requests) || 910 !STAILQ_EMPTY(&sc->sc_queued_requests)) { 911 CESA_UNLOCK(sc, requests); 912 return; 913 } 914 915 /* Move all ready requests to queued list */ 916 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 917 STAILQ_INIT(&sc->sc_ready_requests); 918 919 /* Create one execution chain from all requests on the list */ 920 if (STAILQ_FIRST(&sc->sc_queued_requests) != 921 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 922 prev_cr = NULL; 923 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 924 BUS_DMASYNC_POSTWRITE); 925 926 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 927 if (prev_cr) { 928 ctd = STAILQ_FIRST(&cr->cr_tdesc); 929 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 930 cesa_tdma_desc, ctd_stq); 931 932 prev_ctd->ctd_cthd->cthd_next = 933 ctd->ctd_cthd_paddr; 934 } 935 936 prev_cr = cr; 937 } 938 939 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 940 BUS_DMASYNC_PREWRITE); 941 } 942 943 /* Start chain execution in hardware */ 944 cr = STAILQ_FIRST(&sc->sc_queued_requests); 945 ctd = STAILQ_FIRST(&cr->cr_tdesc); 946 947 CESA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 948 CESA_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 949 950 CESA_UNLOCK(sc, requests); 951 } 952 953 static int 954 cesa_setup_sram(struct cesa_softc *sc) 955 { 956 phandle_t sram_node; 957 ihandle_t sram_ihandle; 958 pcell_t sram_handle, sram_reg; 959 960 if (OF_getprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 961 (void *)&sram_handle, sizeof(sram_handle)) <= 0) 962 return (ENXIO); 963 964 sram_ihandle = (ihandle_t)sram_handle; 965 sram_ihandle = fdt32_to_cpu(sram_ihandle); 966 sram_node = OF_instance_to_package(sram_ihandle); 967 968 if (OF_getprop(sram_node, "reg", (void *)&sram_reg, 969 sizeof(sram_reg)) <= 0) 970 return (ENXIO); 971 972 sc->sc_sram_base = fdt32_to_cpu(sram_reg); 973 974 return (0); 975 } 976 977 static int 978 cesa_probe(device_t dev) 979 { 980 if (!ofw_bus_is_compatible(dev, "mrvl,cesa")) 981 return (ENXIO); 982 983 device_set_desc(dev, "Marvell Cryptographic Engine and Security " 984 "Accelerator"); 985 986 return (BUS_PROBE_DEFAULT); 987 } 988 989 static int 990 cesa_attach(device_t dev) 991 { 992 struct cesa_softc *sc; 993 uint32_t d, r; 994 int error; 995 int i; 996 997 sc = device_get_softc(dev); 998 sc->sc_blocked = 0; 999 sc->sc_error = 0; 1000 sc->sc_dev = dev; 1001 1002 error = cesa_setup_sram(sc); 1003 if (error) { 1004 device_printf(dev, "could not setup SRAM\n"); 1005 return (error); 1006 } 1007 1008 soc_id(&d, &r); 1009 1010 switch (d) { 1011 case MV_DEV_88F6281: 1012 sc->sc_tperr = 0; 1013 break; 1014 case MV_DEV_MV78100: 1015 case MV_DEV_MV78100_Z0: 1016 sc->sc_tperr = CESA_ICR_TPERR; 1017 break; 1018 default: 1019 return (ENXIO); 1020 } 1021 1022 /* Initialize mutexes */ 1023 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1024 "CESA Shared Data", MTX_DEF); 1025 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1026 "CESA TDMA Descriptors Pool", MTX_DEF); 1027 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1028 "CESA SA Descriptors Pool", MTX_DEF); 1029 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1030 "CESA Requests Pool", MTX_DEF); 1031 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1032 "CESA Sessions Pool", MTX_DEF); 1033 1034 /* Allocate I/O and IRQ resources */ 1035 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1036 if (error) { 1037 device_printf(dev, "could not allocate resources\n"); 1038 goto err0; 1039 } 1040 1041 sc->sc_bsh = rman_get_bushandle(*(sc->sc_res)); 1042 sc->sc_bst = rman_get_bustag(*(sc->sc_res)); 1043 1044 /* Setup interrupt handler */ 1045 error = bus_setup_intr(dev, sc->sc_res[1], INTR_TYPE_NET | INTR_MPSAFE, 1046 NULL, cesa_intr, sc, &(sc->sc_icookie)); 1047 if (error) { 1048 device_printf(dev, "could not setup engine completion irq\n"); 1049 goto err1; 1050 } 1051 1052 /* Create DMA tag for processed data */ 1053 error = bus_dma_tag_create(NULL, /* parent */ 1054 1, 0, /* alignment, boundary */ 1055 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1056 BUS_SPACE_MAXADDR, /* highaddr */ 1057 NULL, NULL, /* filtfunc, filtfuncarg */ 1058 CESA_MAX_REQUEST_SIZE, /* maxsize */ 1059 CESA_MAX_FRAGMENTS, /* nsegments */ 1060 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1061 NULL, NULL, /* lockfunc, lockfuncarg */ 1062 &sc->sc_data_dtag); /* dmat */ 1063 if (error) 1064 goto err2; 1065 1066 /* Initialize data structures: TDMA Descriptors Pool */ 1067 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1068 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1069 if (error) 1070 goto err3; 1071 1072 STAILQ_INIT(&sc->sc_free_tdesc); 1073 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1074 sc->sc_tdesc[i].ctd_cthd = 1075 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1076 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1077 (i * sizeof(struct cesa_tdma_hdesc)); 1078 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1079 ctd_stq); 1080 } 1081 1082 /* Initialize data structures: SA Descriptors Pool */ 1083 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1084 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1085 if (error) 1086 goto err4; 1087 1088 STAILQ_INIT(&sc->sc_free_sdesc); 1089 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1090 sc->sc_sdesc[i].csd_cshd = 1091 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1092 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1093 (i * sizeof(struct cesa_sa_hdesc)); 1094 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1095 csd_stq); 1096 } 1097 1098 /* Initialize data structures: Requests Pool */ 1099 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1100 CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1101 if (error) 1102 goto err5; 1103 1104 STAILQ_INIT(&sc->sc_free_requests); 1105 STAILQ_INIT(&sc->sc_ready_requests); 1106 STAILQ_INIT(&sc->sc_queued_requests); 1107 for (i = 0; i < CESA_REQUESTS; i++) { 1108 sc->sc_requests[i].cr_csd = 1109 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1110 sc->sc_requests[i].cr_csd_paddr = 1111 sc->sc_requests_cdm.cdm_paddr + 1112 (i * sizeof(struct cesa_sa_data)); 1113 1114 /* Preallocate DMA maps */ 1115 error = bus_dmamap_create(sc->sc_data_dtag, 0, 1116 &sc->sc_requests[i].cr_dmap); 1117 if (error && i > 0) { 1118 i--; 1119 do { 1120 bus_dmamap_destroy(sc->sc_data_dtag, 1121 sc->sc_requests[i].cr_dmap); 1122 } while (i--); 1123 1124 goto err6; 1125 } 1126 1127 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1128 cr_stq); 1129 } 1130 1131 /* Initialize data structures: Sessions Pool */ 1132 STAILQ_INIT(&sc->sc_free_sessions); 1133 for (i = 0; i < CESA_SESSIONS; i++) { 1134 sc->sc_sessions[i].cs_sid = i; 1135 STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i], 1136 cs_stq); 1137 } 1138 1139 /* 1140 * Initialize TDMA: 1141 * - Burst limit: 128 bytes, 1142 * - Outstanding reads enabled, 1143 * - No byte-swap. 1144 */ 1145 CESA_WRITE(sc, CESA_TDMA_CR, CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 | 1146 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE); 1147 1148 /* 1149 * Initialize SA: 1150 * - SA descriptor is present at beginning of CESA SRAM, 1151 * - Multi-packet chain mode, 1152 * - Cooperation with TDMA enabled. 1153 */ 1154 CESA_WRITE(sc, CESA_SA_DPR, 0); 1155 CESA_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1156 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1157 1158 /* Unmask interrupts */ 1159 CESA_WRITE(sc, CESA_ICR, 0); 1160 CESA_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1161 CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1162 CESA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1163 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1164 CESA_TDMA_EMR_DATA_ERROR); 1165 1166 /* Register in OCF */ 1167 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 1168 if (sc->sc_cid) { 1169 device_printf(dev, "could not get crypto driver id\n"); 1170 goto err7; 1171 } 1172 1173 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 1174 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 1175 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 1176 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 1177 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 1178 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 1179 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 1180 1181 return (0); 1182 err7: 1183 for (i = 0; i < CESA_REQUESTS; i++) 1184 bus_dmamap_destroy(sc->sc_data_dtag, 1185 sc->sc_requests[i].cr_dmap); 1186 err6: 1187 cesa_free_dma_mem(&sc->sc_requests_cdm); 1188 err5: 1189 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1190 err4: 1191 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1192 err3: 1193 bus_dma_tag_destroy(sc->sc_data_dtag); 1194 err2: 1195 bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1196 err1: 1197 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1198 err0: 1199 mtx_destroy(&sc->sc_sessions_lock); 1200 mtx_destroy(&sc->sc_requests_lock); 1201 mtx_destroy(&sc->sc_sdesc_lock); 1202 mtx_destroy(&sc->sc_tdesc_lock); 1203 mtx_destroy(&sc->sc_sc_lock); 1204 return (ENXIO); 1205 } 1206 1207 static int 1208 cesa_detach(device_t dev) 1209 { 1210 struct cesa_softc *sc; 1211 int i; 1212 1213 sc = device_get_softc(dev); 1214 1215 /* TODO: Wait for queued requests completion before shutdown. */ 1216 1217 /* Mask interrupts */ 1218 CESA_WRITE(sc, CESA_ICM, 0); 1219 CESA_WRITE(sc, CESA_TDMA_EMR, 0); 1220 1221 /* Unregister from OCF */ 1222 crypto_unregister_all(sc->sc_cid); 1223 1224 /* Free DMA Maps */ 1225 for (i = 0; i < CESA_REQUESTS; i++) 1226 bus_dmamap_destroy(sc->sc_data_dtag, 1227 sc->sc_requests[i].cr_dmap); 1228 1229 /* Free DMA Memory */ 1230 cesa_free_dma_mem(&sc->sc_requests_cdm); 1231 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1232 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1233 1234 /* Free DMA Tag */ 1235 bus_dma_tag_destroy(sc->sc_data_dtag); 1236 1237 /* Stop interrupt */ 1238 bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1239 1240 /* Relase I/O and IRQ resources */ 1241 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1242 1243 /* Destory mutexes */ 1244 mtx_destroy(&sc->sc_sessions_lock); 1245 mtx_destroy(&sc->sc_requests_lock); 1246 mtx_destroy(&sc->sc_sdesc_lock); 1247 mtx_destroy(&sc->sc_tdesc_lock); 1248 mtx_destroy(&sc->sc_sc_lock); 1249 1250 return (0); 1251 } 1252 1253 static void 1254 cesa_intr(void *arg) 1255 { 1256 STAILQ_HEAD(, cesa_request) requests; 1257 struct cesa_request *cr, *tmp; 1258 struct cesa_softc *sc; 1259 uint32_t ecr, icr; 1260 int blocked; 1261 1262 sc = arg; 1263 1264 /* Ack interrupt */ 1265 ecr = CESA_READ(sc, CESA_TDMA_ECR); 1266 CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1267 icr = CESA_READ(sc, CESA_ICR); 1268 CESA_WRITE(sc, CESA_ICR, 0); 1269 1270 /* Check for TDMA errors */ 1271 if (ecr & CESA_TDMA_ECR_MISS) { 1272 device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1273 sc->sc_error = EIO; 1274 } 1275 1276 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1277 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1278 sc->sc_error = EIO; 1279 } 1280 1281 if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1282 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1283 sc->sc_error = EIO; 1284 } 1285 1286 if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1287 device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1288 sc->sc_error = EIO; 1289 } 1290 1291 /* Check for CESA errors */ 1292 if (icr & sc->sc_tperr) { 1293 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1294 sc->sc_error = EIO; 1295 } 1296 1297 /* If there is nothing more to do, return */ 1298 if ((icr & CESA_ICR_ACCTDMA) == 0) 1299 return; 1300 1301 /* Get all finished requests */ 1302 CESA_LOCK(sc, requests); 1303 STAILQ_INIT(&requests); 1304 STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1305 STAILQ_INIT(&sc->sc_queued_requests); 1306 CESA_UNLOCK(sc, requests); 1307 1308 /* Execute all ready requests */ 1309 cesa_execute(sc); 1310 1311 /* Process completed requests */ 1312 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1313 BUS_DMASYNC_POSTWRITE); 1314 1315 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1316 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1317 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1318 1319 cr->cr_crp->crp_etype = sc->sc_error; 1320 if (cr->cr_mac) 1321 crypto_copyback(cr->cr_crp->crp_flags, 1322 cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, 1323 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1324 1325 crypto_done(cr->cr_crp); 1326 cesa_free_request(sc, cr); 1327 } 1328 1329 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1330 BUS_DMASYNC_PREWRITE); 1331 1332 sc->sc_error = 0; 1333 1334 /* Unblock driver if it ran out of resources */ 1335 CESA_LOCK(sc, sc); 1336 blocked = sc->sc_blocked; 1337 sc->sc_blocked = 0; 1338 CESA_UNLOCK(sc, sc); 1339 1340 if (blocked) 1341 crypto_unblock(sc->sc_cid, blocked); 1342 } 1343 1344 static int 1345 cesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 1346 { 1347 struct cesa_session *cs; 1348 struct cesa_softc *sc; 1349 struct cryptoini *enc; 1350 struct cryptoini *mac; 1351 int error; 1352 1353 sc = device_get_softc(dev); 1354 enc = NULL; 1355 mac = NULL; 1356 error = 0; 1357 1358 /* Check and parse input */ 1359 if (cesa_is_hash(cri->cri_alg)) 1360 mac = cri; 1361 else 1362 enc = cri; 1363 1364 cri = cri->cri_next; 1365 1366 if (cri) { 1367 if (!enc && !cesa_is_hash(cri->cri_alg)) 1368 enc = cri; 1369 1370 if (!mac && cesa_is_hash(cri->cri_alg)) 1371 mac = cri; 1372 1373 if (cri->cri_next || !(enc && mac)) 1374 return (EINVAL); 1375 } 1376 1377 if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || 1378 (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) 1379 return (E2BIG); 1380 1381 /* Allocate session */ 1382 cs = cesa_alloc_session(sc); 1383 if (!cs) 1384 return (ENOMEM); 1385 1386 /* Prepare CESA configuration */ 1387 cs->cs_config = 0; 1388 cs->cs_ivlen = 1; 1389 cs->cs_mblen = 1; 1390 1391 if (enc) { 1392 switch (enc->cri_alg) { 1393 case CRYPTO_AES_CBC: 1394 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1395 cs->cs_ivlen = AES_BLOCK_LEN; 1396 break; 1397 case CRYPTO_DES_CBC: 1398 cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; 1399 cs->cs_ivlen = DES_BLOCK_LEN; 1400 break; 1401 case CRYPTO_3DES_CBC: 1402 cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | 1403 CESA_CSHD_CBC; 1404 cs->cs_ivlen = DES3_BLOCK_LEN; 1405 break; 1406 default: 1407 error = EINVAL; 1408 break; 1409 } 1410 } 1411 1412 if (!error && mac) { 1413 switch (mac->cri_alg) { 1414 case CRYPTO_MD5: 1415 cs->cs_config |= CESA_CSHD_MD5; 1416 cs->cs_mblen = 1; 1417 cs->cs_hlen = MD5_HASH_LEN; 1418 break; 1419 case CRYPTO_MD5_HMAC: 1420 cs->cs_config |= CESA_CSHD_MD5_HMAC; 1421 cs->cs_mblen = MD5_HMAC_BLOCK_LEN; 1422 cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1423 break; 1424 case CRYPTO_SHA1: 1425 cs->cs_config |= CESA_CSHD_SHA1; 1426 cs->cs_mblen = 1; 1427 cs->cs_hlen = SHA1_HASH_LEN; 1428 break; 1429 case CRYPTO_SHA1_HMAC: 1430 cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1431 cs->cs_mblen = SHA1_HMAC_BLOCK_LEN; 1432 cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1433 break; 1434 default: 1435 error = EINVAL; 1436 break; 1437 } 1438 } 1439 1440 /* Save cipher key */ 1441 if (!error && enc && enc->cri_key) { 1442 cs->cs_klen = enc->cri_klen / 8; 1443 memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); 1444 if (enc->cri_alg == CRYPTO_AES_CBC) 1445 error = cesa_prep_aes_key(cs); 1446 } 1447 1448 /* Save digest key */ 1449 if (!error && mac && mac->cri_key) 1450 error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, 1451 mac->cri_klen / 8); 1452 1453 if (error) { 1454 cesa_free_session(sc, cs); 1455 return (EINVAL); 1456 } 1457 1458 *sidp = cs->cs_sid; 1459 1460 return (0); 1461 } 1462 1463 static int 1464 cesa_freesession(device_t dev, uint64_t tid) 1465 { 1466 struct cesa_session *cs; 1467 struct cesa_softc *sc; 1468 1469 sc = device_get_softc(dev); 1470 cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid)); 1471 if (!cs) 1472 return (EINVAL); 1473 1474 /* Free session */ 1475 cesa_free_session(sc, cs); 1476 1477 return (0); 1478 } 1479 1480 static int 1481 cesa_process(device_t dev, struct cryptop *crp, int hint) 1482 { 1483 struct cesa_request *cr; 1484 struct cesa_session *cs; 1485 struct cryptodesc *crd; 1486 struct cryptodesc *enc; 1487 struct cryptodesc *mac; 1488 struct cesa_softc *sc; 1489 int error; 1490 1491 sc = device_get_softc(dev); 1492 crd = crp->crp_desc; 1493 enc = NULL; 1494 mac = NULL; 1495 error = 0; 1496 1497 /* Check session ID */ 1498 cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1499 if (!cs) { 1500 crp->crp_etype = EINVAL; 1501 crypto_done(crp); 1502 return (0); 1503 } 1504 1505 /* Check and parse input */ 1506 if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { 1507 crp->crp_etype = E2BIG; 1508 crypto_done(crp); 1509 return (0); 1510 } 1511 1512 if (cesa_is_hash(crd->crd_alg)) 1513 mac = crd; 1514 else 1515 enc = crd; 1516 1517 crd = crd->crd_next; 1518 1519 if (crd) { 1520 if (!enc && !cesa_is_hash(crd->crd_alg)) 1521 enc = crd; 1522 1523 if (!mac && cesa_is_hash(crd->crd_alg)) 1524 mac = crd; 1525 1526 if (crd->crd_next || !(enc && mac)) { 1527 crp->crp_etype = EINVAL; 1528 crypto_done(crp); 1529 return (0); 1530 } 1531 } 1532 1533 /* 1534 * Get request descriptor. Block driver if there is no free 1535 * descriptors in pool. 1536 */ 1537 cr = cesa_alloc_request(sc); 1538 if (!cr) { 1539 CESA_LOCK(sc, sc); 1540 sc->sc_blocked = CRYPTO_SYMQ; 1541 CESA_UNLOCK(sc, sc); 1542 return (ERESTART); 1543 } 1544 1545 /* Prepare request */ 1546 cr->cr_crp = crp; 1547 cr->cr_enc = enc; 1548 cr->cr_mac = mac; 1549 cr->cr_cs = cs; 1550 1551 CESA_LOCK(sc, sessions); 1552 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1553 1554 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1555 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1556 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1557 else 1558 arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); 1559 1560 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1561 crypto_copyback(crp->crp_flags, crp->crp_buf, 1562 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1563 } else if (enc) { 1564 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1565 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1566 else 1567 crypto_copydata(crp->crp_flags, crp->crp_buf, 1568 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1569 } 1570 1571 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1572 if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { 1573 cs->cs_klen = enc->crd_klen / 8; 1574 memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); 1575 if (enc->crd_alg == CRYPTO_AES_CBC) 1576 error = cesa_prep_aes_key(cs); 1577 } else 1578 error = E2BIG; 1579 } 1580 1581 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1582 if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) 1583 error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, 1584 mac->crd_klen / 8); 1585 else 1586 error = E2BIG; 1587 } 1588 1589 /* Convert request to chain of TDMA and SA descriptors */ 1590 if (!error) 1591 error = cesa_create_chain(sc, cr); 1592 1593 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1594 CESA_UNLOCK(sc, sessions); 1595 1596 if (error) { 1597 cesa_free_request(sc, cr); 1598 crp->crp_etype = error; 1599 crypto_done(crp); 1600 return (0); 1601 } 1602 1603 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1604 BUS_DMASYNC_PREWRITE); 1605 1606 /* Enqueue request to execution */ 1607 cesa_enqueue_request(sc, cr); 1608 1609 /* Start execution, if we have no more requests in queue */ 1610 if ((hint & CRYPTO_HINT_MORE) == 0) 1611 cesa_execute(sc); 1612 1613 return (0); 1614 } 1615