1 /*- 2 * Copyright (C) 2009-2011 Semihalf. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * CESA SRAM Memory Map: 29 * 30 * +------------------------+ <= sc->sc_sram_base + CESA_SRAM_SIZE 31 * | | 32 * | DATA | 33 * | | 34 * +------------------------+ <= sc->sc_sram_base + CESA_DATA(0) 35 * | struct cesa_sa_data | 36 * +------------------------+ 37 * | struct cesa_sa_hdesc | 38 * +------------------------+ <= sc->sc_sram_base 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/kernel.h> 49 #include <sys/lock.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/mutex.h> 53 #include <sys/rman.h> 54 55 #include <machine/bus.h> 56 #include <machine/intr.h> 57 #include <machine/resource.h> 58 59 #include <dev/fdt/fdt_common.h> 60 #include <dev/ofw/ofw_bus.h> 61 #include <dev/ofw/ofw_bus_subr.h> 62 63 #include <sys/md5.h> 64 #include <crypto/sha1.h> 65 #include <crypto/rijndael/rijndael.h> 66 #include <opencrypto/cryptodev.h> 67 #include "cryptodev_if.h" 68 69 #include <arm/mv/mvreg.h> 70 #include <arm/mv/mvwin.h> 71 #include <arm/mv/mvvar.h> 72 #include "cesa.h" 73 74 #undef DEBUG 75 76 static int cesa_probe(device_t); 77 static int cesa_attach(device_t); 78 static int cesa_detach(device_t); 79 static void cesa_intr(void *); 80 static int cesa_newsession(device_t, u_int32_t *, struct cryptoini *); 81 static int cesa_freesession(device_t, u_int64_t); 82 static int cesa_process(device_t, struct cryptop *, int); 83 static int decode_win_cesa_setup(struct cesa_softc *sc); 84 85 static struct resource_spec cesa_res_spec[] = { 86 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 87 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 88 { -1, 0 } 89 }; 90 91 static device_method_t cesa_methods[] = { 92 /* Device interface */ 93 DEVMETHOD(device_probe, cesa_probe), 94 DEVMETHOD(device_attach, cesa_attach), 95 DEVMETHOD(device_detach, cesa_detach), 96 97 /* Crypto device methods */ 98 DEVMETHOD(cryptodev_newsession, cesa_newsession), 99 DEVMETHOD(cryptodev_freesession,cesa_freesession), 100 DEVMETHOD(cryptodev_process, cesa_process), 101 102 DEVMETHOD_END 103 }; 104 105 static driver_t cesa_driver = { 106 "cesa", 107 cesa_methods, 108 sizeof (struct cesa_softc) 109 }; 110 static devclass_t cesa_devclass; 111 112 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); 113 MODULE_DEPEND(cesa, crypto, 1, 1, 1); 114 115 static void 116 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 117 { 118 #ifdef DEBUG 119 device_t dev; 120 121 dev = sc->sc_dev; 122 device_printf(dev, "CESA SA Hardware Descriptor:\n"); 123 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 124 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 125 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 126 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 127 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 128 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 129 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 130 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 131 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 132 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 133 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 134 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 135 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 136 #endif 137 } 138 139 static void 140 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 141 { 142 struct cesa_dma_mem *cdm; 143 144 if (error) 145 return; 146 147 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 148 cdm = arg; 149 cdm->cdm_paddr = segs->ds_addr; 150 } 151 152 static int 153 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 154 bus_size_t size) 155 { 156 int error; 157 158 KASSERT(cdm->cdm_vaddr == NULL, 159 ("%s(): DMA memory descriptor in use.", __func__)); 160 161 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 162 PAGE_SIZE, 0, /* alignment, boundary */ 163 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 164 BUS_SPACE_MAXADDR, /* highaddr */ 165 NULL, NULL, /* filtfunc, filtfuncarg */ 166 size, 1, /* maxsize, nsegments */ 167 size, 0, /* maxsegsz, flags */ 168 NULL, NULL, /* lockfunc, lockfuncarg */ 169 &cdm->cdm_tag); /* dmat */ 170 if (error) { 171 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 172 " %i!\n", error); 173 174 goto err1; 175 } 176 177 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 178 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 179 if (error) { 180 device_printf(sc->sc_dev, "failed to allocate DMA safe" 181 " memory, error %i!\n", error); 182 183 goto err2; 184 } 185 186 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 187 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 188 if (error) { 189 device_printf(sc->sc_dev, "cannot get address of the DMA" 190 " memory, error %i\n", error); 191 192 goto err3; 193 } 194 195 return (0); 196 err3: 197 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 198 err2: 199 bus_dma_tag_destroy(cdm->cdm_tag); 200 err1: 201 cdm->cdm_vaddr = NULL; 202 return (error); 203 } 204 205 static void 206 cesa_free_dma_mem(struct cesa_dma_mem *cdm) 207 { 208 209 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 210 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 211 bus_dma_tag_destroy(cdm->cdm_tag); 212 cdm->cdm_vaddr = NULL; 213 } 214 215 static void 216 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 217 { 218 219 /* Sync only if dma memory is valid */ 220 if (cdm->cdm_vaddr != NULL) 221 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 222 } 223 224 static void 225 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 226 { 227 228 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 229 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 230 cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 231 } 232 233 static struct cesa_session * 234 cesa_alloc_session(struct cesa_softc *sc) 235 { 236 struct cesa_session *cs; 237 238 CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions); 239 240 return (cs); 241 } 242 243 static struct cesa_session * 244 cesa_get_session(struct cesa_softc *sc, uint32_t sid) 245 { 246 247 if (sid >= CESA_SESSIONS) 248 return (NULL); 249 250 return (&sc->sc_sessions[sid]); 251 } 252 253 static void 254 cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs) 255 { 256 257 CESA_GENERIC_FREE_LOCKED(sc, cs, sessions); 258 } 259 260 static struct cesa_request * 261 cesa_alloc_request(struct cesa_softc *sc) 262 { 263 struct cesa_request *cr; 264 265 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 266 if (!cr) 267 return (NULL); 268 269 STAILQ_INIT(&cr->cr_tdesc); 270 STAILQ_INIT(&cr->cr_sdesc); 271 272 return (cr); 273 } 274 275 static void 276 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 277 { 278 279 /* Free TDMA descriptors assigned to this request */ 280 CESA_LOCK(sc, tdesc); 281 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 282 CESA_UNLOCK(sc, tdesc); 283 284 /* Free SA descriptors assigned to this request */ 285 CESA_LOCK(sc, sdesc); 286 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 287 CESA_UNLOCK(sc, sdesc); 288 289 /* Unload DMA memory asociated with request */ 290 if (cr->cr_dmap_loaded) { 291 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 292 cr->cr_dmap_loaded = 0; 293 } 294 295 CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 296 } 297 298 static void 299 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 300 { 301 302 CESA_LOCK(sc, requests); 303 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 304 CESA_UNLOCK(sc, requests); 305 } 306 307 static struct cesa_tdma_desc * 308 cesa_alloc_tdesc(struct cesa_softc *sc) 309 { 310 struct cesa_tdma_desc *ctd; 311 312 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 313 314 if (!ctd) 315 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 316 "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 317 318 return (ctd); 319 } 320 321 static struct cesa_sa_desc * 322 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 323 { 324 struct cesa_sa_desc *csd; 325 326 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 327 if (!csd) { 328 device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 329 "Consider increasing CESA_SA_DESCRIPTORS.\n"); 330 return (NULL); 331 } 332 333 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 334 335 /* Fill-in SA descriptor with default values */ 336 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 337 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 338 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 339 csd->csd_cshd->cshd_enc_src = 0; 340 csd->csd_cshd->cshd_enc_dst = 0; 341 csd->csd_cshd->cshd_enc_dlen = 0; 342 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 343 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 344 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 345 csd->csd_cshd->cshd_mac_src = 0; 346 csd->csd_cshd->cshd_mac_dlen = 0; 347 348 return (csd); 349 } 350 351 static struct cesa_tdma_desc * 352 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 353 bus_size_t size) 354 { 355 struct cesa_tdma_desc *ctd; 356 357 ctd = cesa_alloc_tdesc(sc); 358 if (!ctd) 359 return (NULL); 360 361 ctd->ctd_cthd->cthd_dst = dst; 362 ctd->ctd_cthd->cthd_src = src; 363 ctd->ctd_cthd->cthd_byte_count = size; 364 365 /* Handle special control packet */ 366 if (size != 0) 367 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 368 else 369 ctd->ctd_cthd->cthd_flags = 0; 370 371 return (ctd); 372 } 373 374 static struct cesa_tdma_desc * 375 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 376 { 377 378 return (cesa_tdma_copy(sc, sc->sc_sram_base + 379 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 380 sizeof(struct cesa_sa_data))); 381 } 382 383 static struct cesa_tdma_desc * 384 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 385 { 386 387 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base + 388 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 389 } 390 391 static struct cesa_tdma_desc * 392 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 393 { 394 395 return (cesa_tdma_copy(sc, sc->sc_sram_base, csd->csd_cshd_paddr, 396 sizeof(struct cesa_sa_hdesc))); 397 } 398 399 static void 400 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 401 { 402 struct cesa_tdma_desc *ctd_prev; 403 404 if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 405 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 406 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 407 } 408 409 ctd->ctd_cthd->cthd_next = 0; 410 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 411 } 412 413 static int 414 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 415 struct cesa_packet *cp, struct cesa_sa_desc *csd) 416 { 417 struct cesa_tdma_desc *ctd, *tmp; 418 419 /* Copy SA descriptor for this packet */ 420 ctd = cesa_tdma_copy_sdesc(sc, csd); 421 if (!ctd) 422 return (ENOMEM); 423 424 cesa_append_tdesc(cr, ctd); 425 426 /* Copy data to be processed */ 427 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 428 cesa_append_tdesc(cr, ctd); 429 STAILQ_INIT(&cp->cp_copyin); 430 431 /* Insert control descriptor */ 432 ctd = cesa_tdma_copy(sc, 0, 0, 0); 433 if (!ctd) 434 return (ENOMEM); 435 436 cesa_append_tdesc(cr, ctd); 437 438 /* Copy back results */ 439 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 440 cesa_append_tdesc(cr, ctd); 441 STAILQ_INIT(&cp->cp_copyout); 442 443 return (0); 444 } 445 446 static int 447 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 448 { 449 uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; 450 uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; 451 SHA1_CTX sha1ctx; 452 MD5_CTX md5ctx; 453 uint32_t *hout; 454 uint32_t *hin; 455 int i; 456 457 memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 458 memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 459 for (i = 0; i < mklen; i++) { 460 ipad[i] ^= mkey[i]; 461 opad[i] ^= mkey[i]; 462 } 463 464 hin = (uint32_t *)cs->cs_hiv_in; 465 hout = (uint32_t *)cs->cs_hiv_out; 466 467 switch (alg) { 468 case CRYPTO_MD5_HMAC: 469 MD5Init(&md5ctx); 470 MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN); 471 memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); 472 MD5Init(&md5ctx); 473 MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN); 474 memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); 475 break; 476 case CRYPTO_SHA1_HMAC: 477 SHA1Init(&sha1ctx); 478 SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN); 479 memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 480 SHA1Init(&sha1ctx); 481 SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN); 482 memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 483 break; 484 default: 485 return (EINVAL); 486 } 487 488 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 489 hin[i] = htobe32(hin[i]); 490 hout[i] = htobe32(hout[i]); 491 } 492 493 return (0); 494 } 495 496 static int 497 cesa_prep_aes_key(struct cesa_session *cs) 498 { 499 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 500 uint32_t *dkey; 501 int i; 502 503 rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); 504 505 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 506 dkey = (uint32_t *)cs->cs_aes_dkey; 507 508 switch (cs->cs_klen) { 509 case 16: 510 cs->cs_config |= CESA_CSH_AES_KLEN_128; 511 for (i = 0; i < 4; i++) 512 *dkey++ = htobe32(ek[4 * 10 + i]); 513 break; 514 case 24: 515 cs->cs_config |= CESA_CSH_AES_KLEN_192; 516 for (i = 0; i < 4; i++) 517 *dkey++ = htobe32(ek[4 * 12 + i]); 518 for (i = 0; i < 2; i++) 519 *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 520 break; 521 case 32: 522 cs->cs_config |= CESA_CSH_AES_KLEN_256; 523 for (i = 0; i < 4; i++) 524 *dkey++ = htobe32(ek[4 * 14 + i]); 525 for (i = 0; i < 4; i++) 526 *dkey++ = htobe32(ek[4 * 13 + i]); 527 break; 528 default: 529 return (EINVAL); 530 } 531 532 return (0); 533 } 534 535 static int 536 cesa_is_hash(int alg) 537 { 538 539 switch (alg) { 540 case CRYPTO_MD5: 541 case CRYPTO_MD5_HMAC: 542 case CRYPTO_SHA1: 543 case CRYPTO_SHA1_HMAC: 544 return (1); 545 default: 546 return (0); 547 } 548 } 549 550 static void 551 cesa_start_packet(struct cesa_packet *cp, unsigned int size) 552 { 553 554 cp->cp_size = size; 555 cp->cp_offset = 0; 556 STAILQ_INIT(&cp->cp_copyin); 557 STAILQ_INIT(&cp->cp_copyout); 558 } 559 560 static int 561 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 562 bus_dma_segment_t *seg) 563 { 564 struct cesa_tdma_desc *ctd; 565 unsigned int bsize; 566 567 /* Calculate size of block copy */ 568 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 569 570 if (bsize > 0) { 571 ctd = cesa_tdma_copy(sc, sc->sc_sram_base + 572 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 573 if (!ctd) 574 return (-ENOMEM); 575 576 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 577 578 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base + 579 CESA_DATA(cp->cp_offset), bsize); 580 if (!ctd) 581 return (-ENOMEM); 582 583 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 584 585 seg->ds_len -= bsize; 586 seg->ds_addr += bsize; 587 cp->cp_offset += bsize; 588 } 589 590 return (bsize); 591 } 592 593 static void 594 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 595 { 596 unsigned int mpsize, fragmented; 597 unsigned int mlen, mskip, tmlen; 598 struct cesa_chain_info *cci; 599 unsigned int elen, eskip; 600 unsigned int skip, len; 601 struct cesa_sa_desc *csd; 602 struct cesa_request *cr; 603 struct cesa_softc *sc; 604 struct cesa_packet cp; 605 bus_dma_segment_t seg; 606 uint32_t config; 607 int size; 608 609 cci = arg; 610 sc = cci->cci_sc; 611 cr = cci->cci_cr; 612 613 if (error) { 614 cci->cci_error = error; 615 return; 616 } 617 618 elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; 619 eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; 620 mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; 621 mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; 622 623 if (elen && mlen && 624 ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || 625 (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || 626 (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { 627 /* 628 * Data alignment in the request does not meet CESA requiremnts 629 * for combined encryption/decryption and hashing. We have to 630 * split the request to separate operations and process them 631 * one by one. 632 */ 633 config = cci->cci_config; 634 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 635 config &= ~CESA_CSHD_OP_MASK; 636 637 cci->cci_config = config | CESA_CSHD_MAC; 638 cci->cci_enc = NULL; 639 cci->cci_mac = cr->cr_mac; 640 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 641 642 cci->cci_config = config | CESA_CSHD_ENC; 643 cci->cci_enc = cr->cr_enc; 644 cci->cci_mac = NULL; 645 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 646 } else { 647 config &= ~CESA_CSHD_OP_MASK; 648 649 cci->cci_config = config | CESA_CSHD_ENC; 650 cci->cci_enc = cr->cr_enc; 651 cci->cci_mac = NULL; 652 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 653 654 cci->cci_config = config | CESA_CSHD_MAC; 655 cci->cci_enc = NULL; 656 cci->cci_mac = cr->cr_mac; 657 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 658 } 659 660 return; 661 } 662 663 tmlen = mlen; 664 fragmented = 0; 665 mpsize = CESA_MAX_PACKET_SIZE; 666 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 667 668 if (elen && mlen) { 669 skip = MIN(eskip, mskip); 670 len = MAX(elen + eskip, mlen + mskip) - skip; 671 } else if (elen) { 672 skip = eskip; 673 len = elen; 674 } else { 675 skip = mskip; 676 len = mlen; 677 } 678 679 /* Start first packet in chain */ 680 cesa_start_packet(&cp, MIN(mpsize, len)); 681 682 while (nseg-- && len > 0) { 683 seg = *(segs++); 684 685 /* 686 * Skip data in buffer on which neither ENC nor MAC operation 687 * is requested. 688 */ 689 if (skip > 0) { 690 size = MIN(skip, seg.ds_len); 691 skip -= size; 692 693 seg.ds_addr += size; 694 seg.ds_len -= size; 695 696 if (eskip > 0) 697 eskip -= size; 698 699 if (mskip > 0) 700 mskip -= size; 701 702 if (seg.ds_len == 0) 703 continue; 704 } 705 706 while (1) { 707 /* 708 * Fill in current packet with data. Break if there is 709 * no more data in current DMA segment or an error 710 * occured. 711 */ 712 size = cesa_fill_packet(sc, &cp, &seg); 713 if (size <= 0) { 714 error = -size; 715 break; 716 } 717 718 len -= size; 719 720 /* If packet is full, append it to the chain */ 721 if (cp.cp_size == cp.cp_offset) { 722 csd = cesa_alloc_sdesc(sc, cr); 723 if (!csd) { 724 error = ENOMEM; 725 break; 726 } 727 728 /* Create SA descriptor for this packet */ 729 csd->csd_cshd->cshd_config = cci->cci_config; 730 csd->csd_cshd->cshd_mac_total_dlen = tmlen; 731 732 /* 733 * Enable fragmentation if request will not fit 734 * into one packet. 735 */ 736 if (len > 0) { 737 if (!fragmented) { 738 fragmented = 1; 739 csd->csd_cshd->cshd_config |= 740 CESA_CSHD_FRAG_FIRST; 741 } else 742 csd->csd_cshd->cshd_config |= 743 CESA_CSHD_FRAG_MIDDLE; 744 } else if (fragmented) 745 csd->csd_cshd->cshd_config |= 746 CESA_CSHD_FRAG_LAST; 747 748 if (eskip < cp.cp_size && elen > 0) { 749 csd->csd_cshd->cshd_enc_src = 750 CESA_DATA(eskip); 751 csd->csd_cshd->cshd_enc_dst = 752 CESA_DATA(eskip); 753 csd->csd_cshd->cshd_enc_dlen = 754 MIN(elen, cp.cp_size - eskip); 755 } 756 757 if (mskip < cp.cp_size && mlen > 0) { 758 csd->csd_cshd->cshd_mac_src = 759 CESA_DATA(mskip); 760 csd->csd_cshd->cshd_mac_dlen = 761 MIN(mlen, cp.cp_size - mskip); 762 } 763 764 elen -= csd->csd_cshd->cshd_enc_dlen; 765 eskip -= MIN(eskip, cp.cp_size); 766 mlen -= csd->csd_cshd->cshd_mac_dlen; 767 mskip -= MIN(mskip, cp.cp_size); 768 769 cesa_dump_cshd(sc, csd->csd_cshd); 770 771 /* Append packet to the request */ 772 error = cesa_append_packet(sc, cr, &cp, csd); 773 if (error) 774 break; 775 776 /* Start a new packet, as current is full */ 777 cesa_start_packet(&cp, MIN(mpsize, len)); 778 } 779 } 780 781 if (error) 782 break; 783 } 784 785 if (error) { 786 /* 787 * Move all allocated resources to the request. They will be 788 * freed later. 789 */ 790 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 791 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 792 cci->cci_error = error; 793 } 794 } 795 796 static void 797 cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 798 bus_size_t size, int error) 799 { 800 801 cesa_create_chain_cb(arg, segs, nseg, error); 802 } 803 804 static int 805 cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) 806 { 807 struct cesa_chain_info cci; 808 struct cesa_tdma_desc *ctd; 809 uint32_t config; 810 int error; 811 812 error = 0; 813 CESA_LOCK_ASSERT(sc, sessions); 814 815 /* Create request metadata */ 816 if (cr->cr_enc) { 817 if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && 818 (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 819 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 820 cr->cr_cs->cs_klen); 821 else 822 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 823 cr->cr_cs->cs_klen); 824 } 825 826 if (cr->cr_mac) { 827 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 828 CESA_MAX_HASH_LEN); 829 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 830 CESA_MAX_HASH_LEN); 831 } 832 833 ctd = cesa_tdma_copyin_sa_data(sc, cr); 834 if (!ctd) 835 return (ENOMEM); 836 837 cesa_append_tdesc(cr, ctd); 838 839 /* Prepare SA configuration */ 840 config = cr->cr_cs->cs_config; 841 842 if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 843 config |= CESA_CSHD_DECRYPT; 844 if (cr->cr_enc && !cr->cr_mac) 845 config |= CESA_CSHD_ENC; 846 if (!cr->cr_enc && cr->cr_mac) 847 config |= CESA_CSHD_MAC; 848 if (cr->cr_enc && cr->cr_mac) 849 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 850 CESA_CSHD_ENC_AND_MAC; 851 852 /* Create data packets */ 853 cci.cci_sc = sc; 854 cci.cci_cr = cr; 855 cci.cci_enc = cr->cr_enc; 856 cci.cci_mac = cr->cr_mac; 857 cci.cci_config = config; 858 cci.cci_error = 0; 859 860 if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) 861 error = bus_dmamap_load_uio(sc->sc_data_dtag, 862 cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, 863 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 864 else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) 865 error = bus_dmamap_load_mbuf(sc->sc_data_dtag, 866 cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, 867 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 868 else 869 error = bus_dmamap_load(sc->sc_data_dtag, 870 cr->cr_dmap, cr->cr_crp->crp_buf, 871 cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, 872 BUS_DMA_NOWAIT); 873 874 if (!error) 875 cr->cr_dmap_loaded = 1; 876 877 if (cci.cci_error) 878 error = cci.cci_error; 879 880 if (error) 881 return (error); 882 883 /* Read back request metadata */ 884 ctd = cesa_tdma_copyout_sa_data(sc, cr); 885 if (!ctd) 886 return (ENOMEM); 887 888 cesa_append_tdesc(cr, ctd); 889 890 return (0); 891 } 892 893 static void 894 cesa_execute(struct cesa_softc *sc) 895 { 896 struct cesa_tdma_desc *prev_ctd, *ctd; 897 struct cesa_request *prev_cr, *cr; 898 899 CESA_LOCK(sc, requests); 900 901 /* 902 * If ready list is empty, there is nothing to execute. If queued list 903 * is not empty, the hardware is busy and we cannot start another 904 * execution. 905 */ 906 if (STAILQ_EMPTY(&sc->sc_ready_requests) || 907 !STAILQ_EMPTY(&sc->sc_queued_requests)) { 908 CESA_UNLOCK(sc, requests); 909 return; 910 } 911 912 /* Move all ready requests to queued list */ 913 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 914 STAILQ_INIT(&sc->sc_ready_requests); 915 916 /* Create one execution chain from all requests on the list */ 917 if (STAILQ_FIRST(&sc->sc_queued_requests) != 918 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 919 prev_cr = NULL; 920 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 921 BUS_DMASYNC_POSTWRITE); 922 923 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 924 if (prev_cr) { 925 ctd = STAILQ_FIRST(&cr->cr_tdesc); 926 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 927 cesa_tdma_desc, ctd_stq); 928 929 prev_ctd->ctd_cthd->cthd_next = 930 ctd->ctd_cthd_paddr; 931 } 932 933 prev_cr = cr; 934 } 935 936 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 937 BUS_DMASYNC_PREWRITE); 938 } 939 940 /* Start chain execution in hardware */ 941 cr = STAILQ_FIRST(&sc->sc_queued_requests); 942 ctd = STAILQ_FIRST(&cr->cr_tdesc); 943 944 CESA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 945 CESA_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 946 947 CESA_UNLOCK(sc, requests); 948 } 949 950 static int 951 cesa_setup_sram(struct cesa_softc *sc) 952 { 953 phandle_t sram_node; 954 ihandle_t sram_ihandle; 955 pcell_t sram_handle, sram_reg; 956 957 if (OF_getprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 958 (void *)&sram_handle, sizeof(sram_handle)) <= 0) 959 return (ENXIO); 960 961 sram_ihandle = (ihandle_t)sram_handle; 962 sram_ihandle = fdt32_to_cpu(sram_ihandle); 963 sram_node = OF_instance_to_package(sram_ihandle); 964 965 if (OF_getprop(sram_node, "reg", (void *)&sram_reg, 966 sizeof(sram_reg)) <= 0) 967 return (ENXIO); 968 969 sc->sc_sram_base = fdt32_to_cpu(sram_reg); 970 971 return (0); 972 } 973 974 static int 975 cesa_probe(device_t dev) 976 { 977 if (!ofw_bus_is_compatible(dev, "mrvl,cesa")) 978 return (ENXIO); 979 980 device_set_desc(dev, "Marvell Cryptographic Engine and Security " 981 "Accelerator"); 982 983 return (BUS_PROBE_DEFAULT); 984 } 985 986 static int 987 cesa_attach(device_t dev) 988 { 989 struct cesa_softc *sc; 990 uint32_t d, r; 991 int error; 992 int i; 993 994 sc = device_get_softc(dev); 995 sc->sc_blocked = 0; 996 sc->sc_error = 0; 997 sc->sc_dev = dev; 998 999 /* Check if CESA peripheral device has power turned on */ 1000 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) != CPU_PM_CTRL_CRYPTO) { 1001 device_printf(dev, "not powered on\n"); 1002 return (ENXIO); 1003 } 1004 1005 soc_id(&d, &r); 1006 1007 switch (d) { 1008 case MV_DEV_88F6281: 1009 case MV_DEV_88F6282: 1010 sc->sc_tperr = 0; 1011 break; 1012 case MV_DEV_MV78100: 1013 case MV_DEV_MV78100_Z0: 1014 sc->sc_tperr = CESA_ICR_TPERR; 1015 break; 1016 default: 1017 return (ENXIO); 1018 } 1019 1020 /* Initialize mutexes */ 1021 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1022 "CESA Shared Data", MTX_DEF); 1023 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1024 "CESA TDMA Descriptors Pool", MTX_DEF); 1025 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1026 "CESA SA Descriptors Pool", MTX_DEF); 1027 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1028 "CESA Requests Pool", MTX_DEF); 1029 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1030 "CESA Sessions Pool", MTX_DEF); 1031 1032 /* Allocate I/O and IRQ resources */ 1033 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1034 if (error) { 1035 device_printf(dev, "could not allocate resources\n"); 1036 goto err0; 1037 } 1038 1039 sc->sc_bsh = rman_get_bushandle(*(sc->sc_res)); 1040 sc->sc_bst = rman_get_bustag(*(sc->sc_res)); 1041 1042 /* Setup CESA decoding windows */ 1043 error = decode_win_cesa_setup(sc); 1044 if (error) { 1045 device_printf(dev, "could not setup decoding windows\n"); 1046 goto err1; 1047 } 1048 1049 /* Acquire SRAM base address */ 1050 error = cesa_setup_sram(sc); 1051 if (error) { 1052 device_printf(dev, "could not setup SRAM\n"); 1053 goto err1; 1054 } 1055 1056 /* Setup interrupt handler */ 1057 error = bus_setup_intr(dev, sc->sc_res[1], INTR_TYPE_NET | INTR_MPSAFE, 1058 NULL, cesa_intr, sc, &(sc->sc_icookie)); 1059 if (error) { 1060 device_printf(dev, "could not setup engine completion irq\n"); 1061 goto err1; 1062 } 1063 1064 /* Create DMA tag for processed data */ 1065 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1066 1, 0, /* alignment, boundary */ 1067 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1068 BUS_SPACE_MAXADDR, /* highaddr */ 1069 NULL, NULL, /* filtfunc, filtfuncarg */ 1070 CESA_MAX_REQUEST_SIZE, /* maxsize */ 1071 CESA_MAX_FRAGMENTS, /* nsegments */ 1072 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1073 NULL, NULL, /* lockfunc, lockfuncarg */ 1074 &sc->sc_data_dtag); /* dmat */ 1075 if (error) 1076 goto err2; 1077 1078 /* Initialize data structures: TDMA Descriptors Pool */ 1079 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1080 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1081 if (error) 1082 goto err3; 1083 1084 STAILQ_INIT(&sc->sc_free_tdesc); 1085 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1086 sc->sc_tdesc[i].ctd_cthd = 1087 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1088 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1089 (i * sizeof(struct cesa_tdma_hdesc)); 1090 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1091 ctd_stq); 1092 } 1093 1094 /* Initialize data structures: SA Descriptors Pool */ 1095 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1096 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1097 if (error) 1098 goto err4; 1099 1100 STAILQ_INIT(&sc->sc_free_sdesc); 1101 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1102 sc->sc_sdesc[i].csd_cshd = 1103 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1104 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1105 (i * sizeof(struct cesa_sa_hdesc)); 1106 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1107 csd_stq); 1108 } 1109 1110 /* Initialize data structures: Requests Pool */ 1111 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1112 CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1113 if (error) 1114 goto err5; 1115 1116 STAILQ_INIT(&sc->sc_free_requests); 1117 STAILQ_INIT(&sc->sc_ready_requests); 1118 STAILQ_INIT(&sc->sc_queued_requests); 1119 for (i = 0; i < CESA_REQUESTS; i++) { 1120 sc->sc_requests[i].cr_csd = 1121 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1122 sc->sc_requests[i].cr_csd_paddr = 1123 sc->sc_requests_cdm.cdm_paddr + 1124 (i * sizeof(struct cesa_sa_data)); 1125 1126 /* Preallocate DMA maps */ 1127 error = bus_dmamap_create(sc->sc_data_dtag, 0, 1128 &sc->sc_requests[i].cr_dmap); 1129 if (error && i > 0) { 1130 i--; 1131 do { 1132 bus_dmamap_destroy(sc->sc_data_dtag, 1133 sc->sc_requests[i].cr_dmap); 1134 } while (i--); 1135 1136 goto err6; 1137 } 1138 1139 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1140 cr_stq); 1141 } 1142 1143 /* Initialize data structures: Sessions Pool */ 1144 STAILQ_INIT(&sc->sc_free_sessions); 1145 for (i = 0; i < CESA_SESSIONS; i++) { 1146 sc->sc_sessions[i].cs_sid = i; 1147 STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i], 1148 cs_stq); 1149 } 1150 1151 /* 1152 * Initialize TDMA: 1153 * - Burst limit: 128 bytes, 1154 * - Outstanding reads enabled, 1155 * - No byte-swap. 1156 */ 1157 CESA_WRITE(sc, CESA_TDMA_CR, CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 | 1158 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE); 1159 1160 /* 1161 * Initialize SA: 1162 * - SA descriptor is present at beginning of CESA SRAM, 1163 * - Multi-packet chain mode, 1164 * - Cooperation with TDMA enabled. 1165 */ 1166 CESA_WRITE(sc, CESA_SA_DPR, 0); 1167 CESA_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1168 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1169 1170 /* Unmask interrupts */ 1171 CESA_WRITE(sc, CESA_ICR, 0); 1172 CESA_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1173 CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1174 CESA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1175 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1176 CESA_TDMA_EMR_DATA_ERROR); 1177 1178 /* Register in OCF */ 1179 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 1180 if (sc->sc_cid) { 1181 device_printf(dev, "could not get crypto driver id\n"); 1182 goto err7; 1183 } 1184 1185 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 1186 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 1187 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 1188 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 1189 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 1190 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 1191 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 1192 1193 return (0); 1194 err7: 1195 for (i = 0; i < CESA_REQUESTS; i++) 1196 bus_dmamap_destroy(sc->sc_data_dtag, 1197 sc->sc_requests[i].cr_dmap); 1198 err6: 1199 cesa_free_dma_mem(&sc->sc_requests_cdm); 1200 err5: 1201 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1202 err4: 1203 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1204 err3: 1205 bus_dma_tag_destroy(sc->sc_data_dtag); 1206 err2: 1207 bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1208 err1: 1209 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1210 err0: 1211 mtx_destroy(&sc->sc_sessions_lock); 1212 mtx_destroy(&sc->sc_requests_lock); 1213 mtx_destroy(&sc->sc_sdesc_lock); 1214 mtx_destroy(&sc->sc_tdesc_lock); 1215 mtx_destroy(&sc->sc_sc_lock); 1216 return (ENXIO); 1217 } 1218 1219 static int 1220 cesa_detach(device_t dev) 1221 { 1222 struct cesa_softc *sc; 1223 int i; 1224 1225 sc = device_get_softc(dev); 1226 1227 /* TODO: Wait for queued requests completion before shutdown. */ 1228 1229 /* Mask interrupts */ 1230 CESA_WRITE(sc, CESA_ICM, 0); 1231 CESA_WRITE(sc, CESA_TDMA_EMR, 0); 1232 1233 /* Unregister from OCF */ 1234 crypto_unregister_all(sc->sc_cid); 1235 1236 /* Free DMA Maps */ 1237 for (i = 0; i < CESA_REQUESTS; i++) 1238 bus_dmamap_destroy(sc->sc_data_dtag, 1239 sc->sc_requests[i].cr_dmap); 1240 1241 /* Free DMA Memory */ 1242 cesa_free_dma_mem(&sc->sc_requests_cdm); 1243 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1244 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1245 1246 /* Free DMA Tag */ 1247 bus_dma_tag_destroy(sc->sc_data_dtag); 1248 1249 /* Stop interrupt */ 1250 bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1251 1252 /* Relase I/O and IRQ resources */ 1253 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1254 1255 /* Destory mutexes */ 1256 mtx_destroy(&sc->sc_sessions_lock); 1257 mtx_destroy(&sc->sc_requests_lock); 1258 mtx_destroy(&sc->sc_sdesc_lock); 1259 mtx_destroy(&sc->sc_tdesc_lock); 1260 mtx_destroy(&sc->sc_sc_lock); 1261 1262 return (0); 1263 } 1264 1265 static void 1266 cesa_intr(void *arg) 1267 { 1268 STAILQ_HEAD(, cesa_request) requests; 1269 struct cesa_request *cr, *tmp; 1270 struct cesa_softc *sc; 1271 uint32_t ecr, icr; 1272 int blocked; 1273 1274 sc = arg; 1275 1276 /* Ack interrupt */ 1277 ecr = CESA_READ(sc, CESA_TDMA_ECR); 1278 CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1279 icr = CESA_READ(sc, CESA_ICR); 1280 CESA_WRITE(sc, CESA_ICR, 0); 1281 1282 /* Check for TDMA errors */ 1283 if (ecr & CESA_TDMA_ECR_MISS) { 1284 device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1285 sc->sc_error = EIO; 1286 } 1287 1288 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1289 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1290 sc->sc_error = EIO; 1291 } 1292 1293 if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1294 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1295 sc->sc_error = EIO; 1296 } 1297 1298 if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1299 device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1300 sc->sc_error = EIO; 1301 } 1302 1303 /* Check for CESA errors */ 1304 if (icr & sc->sc_tperr) { 1305 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1306 sc->sc_error = EIO; 1307 } 1308 1309 /* If there is nothing more to do, return */ 1310 if ((icr & CESA_ICR_ACCTDMA) == 0) 1311 return; 1312 1313 /* Get all finished requests */ 1314 CESA_LOCK(sc, requests); 1315 STAILQ_INIT(&requests); 1316 STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1317 STAILQ_INIT(&sc->sc_queued_requests); 1318 CESA_UNLOCK(sc, requests); 1319 1320 /* Execute all ready requests */ 1321 cesa_execute(sc); 1322 1323 /* Process completed requests */ 1324 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1325 BUS_DMASYNC_POSTWRITE); 1326 1327 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1328 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1329 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1330 1331 cr->cr_crp->crp_etype = sc->sc_error; 1332 if (cr->cr_mac) 1333 crypto_copyback(cr->cr_crp->crp_flags, 1334 cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, 1335 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1336 1337 crypto_done(cr->cr_crp); 1338 cesa_free_request(sc, cr); 1339 } 1340 1341 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1342 BUS_DMASYNC_PREWRITE); 1343 1344 sc->sc_error = 0; 1345 1346 /* Unblock driver if it ran out of resources */ 1347 CESA_LOCK(sc, sc); 1348 blocked = sc->sc_blocked; 1349 sc->sc_blocked = 0; 1350 CESA_UNLOCK(sc, sc); 1351 1352 if (blocked) 1353 crypto_unblock(sc->sc_cid, blocked); 1354 } 1355 1356 static int 1357 cesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 1358 { 1359 struct cesa_session *cs; 1360 struct cesa_softc *sc; 1361 struct cryptoini *enc; 1362 struct cryptoini *mac; 1363 int error; 1364 1365 sc = device_get_softc(dev); 1366 enc = NULL; 1367 mac = NULL; 1368 error = 0; 1369 1370 /* Check and parse input */ 1371 if (cesa_is_hash(cri->cri_alg)) 1372 mac = cri; 1373 else 1374 enc = cri; 1375 1376 cri = cri->cri_next; 1377 1378 if (cri) { 1379 if (!enc && !cesa_is_hash(cri->cri_alg)) 1380 enc = cri; 1381 1382 if (!mac && cesa_is_hash(cri->cri_alg)) 1383 mac = cri; 1384 1385 if (cri->cri_next || !(enc && mac)) 1386 return (EINVAL); 1387 } 1388 1389 if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || 1390 (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) 1391 return (E2BIG); 1392 1393 /* Allocate session */ 1394 cs = cesa_alloc_session(sc); 1395 if (!cs) 1396 return (ENOMEM); 1397 1398 /* Prepare CESA configuration */ 1399 cs->cs_config = 0; 1400 cs->cs_ivlen = 1; 1401 cs->cs_mblen = 1; 1402 1403 if (enc) { 1404 switch (enc->cri_alg) { 1405 case CRYPTO_AES_CBC: 1406 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1407 cs->cs_ivlen = AES_BLOCK_LEN; 1408 break; 1409 case CRYPTO_DES_CBC: 1410 cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; 1411 cs->cs_ivlen = DES_BLOCK_LEN; 1412 break; 1413 case CRYPTO_3DES_CBC: 1414 cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | 1415 CESA_CSHD_CBC; 1416 cs->cs_ivlen = DES3_BLOCK_LEN; 1417 break; 1418 default: 1419 error = EINVAL; 1420 break; 1421 } 1422 } 1423 1424 if (!error && mac) { 1425 switch (mac->cri_alg) { 1426 case CRYPTO_MD5: 1427 cs->cs_config |= CESA_CSHD_MD5; 1428 cs->cs_mblen = 1; 1429 cs->cs_hlen = MD5_HASH_LEN; 1430 break; 1431 case CRYPTO_MD5_HMAC: 1432 cs->cs_config |= CESA_CSHD_MD5_HMAC; 1433 cs->cs_mblen = MD5_HMAC_BLOCK_LEN; 1434 cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1435 break; 1436 case CRYPTO_SHA1: 1437 cs->cs_config |= CESA_CSHD_SHA1; 1438 cs->cs_mblen = 1; 1439 cs->cs_hlen = SHA1_HASH_LEN; 1440 break; 1441 case CRYPTO_SHA1_HMAC: 1442 cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1443 cs->cs_mblen = SHA1_HMAC_BLOCK_LEN; 1444 cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1445 break; 1446 default: 1447 error = EINVAL; 1448 break; 1449 } 1450 } 1451 1452 /* Save cipher key */ 1453 if (!error && enc && enc->cri_key) { 1454 cs->cs_klen = enc->cri_klen / 8; 1455 memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); 1456 if (enc->cri_alg == CRYPTO_AES_CBC) 1457 error = cesa_prep_aes_key(cs); 1458 } 1459 1460 /* Save digest key */ 1461 if (!error && mac && mac->cri_key) 1462 error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, 1463 mac->cri_klen / 8); 1464 1465 if (error) { 1466 cesa_free_session(sc, cs); 1467 return (EINVAL); 1468 } 1469 1470 *sidp = cs->cs_sid; 1471 1472 return (0); 1473 } 1474 1475 static int 1476 cesa_freesession(device_t dev, uint64_t tid) 1477 { 1478 struct cesa_session *cs; 1479 struct cesa_softc *sc; 1480 1481 sc = device_get_softc(dev); 1482 cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid)); 1483 if (!cs) 1484 return (EINVAL); 1485 1486 /* Free session */ 1487 cesa_free_session(sc, cs); 1488 1489 return (0); 1490 } 1491 1492 static int 1493 cesa_process(device_t dev, struct cryptop *crp, int hint) 1494 { 1495 struct cesa_request *cr; 1496 struct cesa_session *cs; 1497 struct cryptodesc *crd; 1498 struct cryptodesc *enc; 1499 struct cryptodesc *mac; 1500 struct cesa_softc *sc; 1501 int error; 1502 1503 sc = device_get_softc(dev); 1504 crd = crp->crp_desc; 1505 enc = NULL; 1506 mac = NULL; 1507 error = 0; 1508 1509 /* Check session ID */ 1510 cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1511 if (!cs) { 1512 crp->crp_etype = EINVAL; 1513 crypto_done(crp); 1514 return (0); 1515 } 1516 1517 /* Check and parse input */ 1518 if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { 1519 crp->crp_etype = E2BIG; 1520 crypto_done(crp); 1521 return (0); 1522 } 1523 1524 if (cesa_is_hash(crd->crd_alg)) 1525 mac = crd; 1526 else 1527 enc = crd; 1528 1529 crd = crd->crd_next; 1530 1531 if (crd) { 1532 if (!enc && !cesa_is_hash(crd->crd_alg)) 1533 enc = crd; 1534 1535 if (!mac && cesa_is_hash(crd->crd_alg)) 1536 mac = crd; 1537 1538 if (crd->crd_next || !(enc && mac)) { 1539 crp->crp_etype = EINVAL; 1540 crypto_done(crp); 1541 return (0); 1542 } 1543 } 1544 1545 /* 1546 * Get request descriptor. Block driver if there is no free 1547 * descriptors in pool. 1548 */ 1549 cr = cesa_alloc_request(sc); 1550 if (!cr) { 1551 CESA_LOCK(sc, sc); 1552 sc->sc_blocked = CRYPTO_SYMQ; 1553 CESA_UNLOCK(sc, sc); 1554 return (ERESTART); 1555 } 1556 1557 /* Prepare request */ 1558 cr->cr_crp = crp; 1559 cr->cr_enc = enc; 1560 cr->cr_mac = mac; 1561 cr->cr_cs = cs; 1562 1563 CESA_LOCK(sc, sessions); 1564 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1565 1566 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1567 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1568 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1569 else 1570 arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); 1571 1572 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1573 crypto_copyback(crp->crp_flags, crp->crp_buf, 1574 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1575 } else if (enc) { 1576 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1577 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1578 else 1579 crypto_copydata(crp->crp_flags, crp->crp_buf, 1580 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1581 } 1582 1583 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1584 if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { 1585 cs->cs_klen = enc->crd_klen / 8; 1586 memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); 1587 if (enc->crd_alg == CRYPTO_AES_CBC) 1588 error = cesa_prep_aes_key(cs); 1589 } else 1590 error = E2BIG; 1591 } 1592 1593 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1594 if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) 1595 error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, 1596 mac->crd_klen / 8); 1597 else 1598 error = E2BIG; 1599 } 1600 1601 /* Convert request to chain of TDMA and SA descriptors */ 1602 if (!error) 1603 error = cesa_create_chain(sc, cr); 1604 1605 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1606 CESA_UNLOCK(sc, sessions); 1607 1608 if (error) { 1609 cesa_free_request(sc, cr); 1610 crp->crp_etype = error; 1611 crypto_done(crp); 1612 return (0); 1613 } 1614 1615 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1616 BUS_DMASYNC_PREWRITE); 1617 1618 /* Enqueue request to execution */ 1619 cesa_enqueue_request(sc, cr); 1620 1621 /* Start execution, if we have no more requests in queue */ 1622 if ((hint & CRYPTO_HINT_MORE) == 0) 1623 cesa_execute(sc); 1624 1625 return (0); 1626 } 1627 1628 /* 1629 * Set CESA TDMA decode windows. 1630 */ 1631 static int 1632 decode_win_cesa_setup(struct cesa_softc *sc) 1633 { 1634 struct mem_region availmem_regions[FDT_MEM_REGIONS]; 1635 int availmem_regions_sz; 1636 uint32_t memsize, br, cr, i; 1637 1638 /* Grab physical memory regions information from DTS */ 1639 if (fdt_get_mem_regions(availmem_regions, &availmem_regions_sz, 1640 &memsize) != 0) 1641 return (ENXIO); 1642 1643 if (availmem_regions_sz > MV_WIN_CESA_MAX) { 1644 device_printf(sc->sc_dev, "Too much memory regions, cannot " 1645 " set CESA windows to cover whole DRAM \n"); 1646 return (ENXIO); 1647 } 1648 1649 /* Disable and clear all CESA windows */ 1650 for (i = 0; i < MV_WIN_CESA_MAX; i++) { 1651 CESA_WRITE(sc, MV_WIN_CESA_BASE(i), 0); 1652 CESA_WRITE(sc, MV_WIN_CESA_CTRL(i), 0); 1653 } 1654 1655 /* Fill CESA TDMA decoding windows with information acquired from DTS */ 1656 for (i = 0; i < availmem_regions_sz; i++) { 1657 br = availmem_regions[i].mr_start; 1658 cr = availmem_regions[i].mr_size; 1659 1660 /* Don't add entries with size lower than 64KB */ 1661 if (cr & 0xffff0000) { 1662 cr = (((cr - 1) & 0xffff0000) | 1663 (MV_WIN_DDR_ATTR(i) << MV_WIN_CPU_ATTR_SHIFT) | 1664 (MV_WIN_DDR_TARGET << MV_WIN_CPU_TARGET_SHIFT) | 1665 MV_WIN_CPU_ENABLE_BIT); 1666 CESA_WRITE(sc, MV_WIN_CESA_BASE(i), br); 1667 CESA_WRITE(sc, MV_WIN_CESA_CTRL(i), cr); 1668 } 1669 } 1670 1671 return (0); 1672 } 1673 1674