1 /*- 2 * Copyright (C) 2009-2011 Semihalf. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * CESA SRAM Memory Map: 29 * 30 * +------------------------+ <= sc->sc_sram_base + CESA_SRAM_SIZE 31 * | | 32 * | DATA | 33 * | | 34 * +------------------------+ <= sc->sc_sram_base + CESA_DATA(0) 35 * | struct cesa_sa_data | 36 * +------------------------+ 37 * | struct cesa_sa_hdesc | 38 * +------------------------+ <= sc->sc_sram_base 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/kernel.h> 49 #include <sys/lock.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/mutex.h> 53 #include <sys/rman.h> 54 55 #include <machine/bus.h> 56 #include <machine/intr.h> 57 #include <machine/resource.h> 58 59 #include <dev/fdt/fdt_common.h> 60 #include <dev/ofw/ofw_bus.h> 61 #include <dev/ofw/ofw_bus_subr.h> 62 63 #include <sys/md5.h> 64 #include <crypto/sha1.h> 65 #include <crypto/rijndael/rijndael.h> 66 #include <opencrypto/cryptodev.h> 67 #include "cryptodev_if.h" 68 69 #include <arm/mv/mvreg.h> 70 #include <arm/mv/mvwin.h> 71 #include <arm/mv/mvvar.h> 72 #include "cesa.h" 73 74 static int cesa_probe(device_t); 75 static int cesa_attach(device_t); 76 static int cesa_detach(device_t); 77 static void cesa_intr(void *); 78 static int cesa_newsession(device_t, u_int32_t *, struct cryptoini *); 79 static int cesa_freesession(device_t, u_int64_t); 80 static int cesa_process(device_t, struct cryptop *, int); 81 static int decode_win_cesa_setup(struct cesa_softc *sc); 82 83 static struct resource_spec cesa_res_spec[] = { 84 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 85 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 86 { -1, 0 } 87 }; 88 89 static device_method_t cesa_methods[] = { 90 /* Device interface */ 91 DEVMETHOD(device_probe, cesa_probe), 92 DEVMETHOD(device_attach, cesa_attach), 93 DEVMETHOD(device_detach, cesa_detach), 94 95 /* Crypto device methods */ 96 DEVMETHOD(cryptodev_newsession, cesa_newsession), 97 DEVMETHOD(cryptodev_freesession,cesa_freesession), 98 DEVMETHOD(cryptodev_process, cesa_process), 99 100 DEVMETHOD_END 101 }; 102 103 static driver_t cesa_driver = { 104 "cesa", 105 cesa_methods, 106 sizeof (struct cesa_softc) 107 }; 108 static devclass_t cesa_devclass; 109 110 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); 111 MODULE_DEPEND(cesa, crypto, 1, 1, 1); 112 113 static void 114 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 115 { 116 #ifdef DEBUG 117 device_t dev; 118 119 dev = sc->sc_dev; 120 device_printf(dev, "CESA SA Hardware Descriptor:\n"); 121 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 122 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 123 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 124 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 125 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 126 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 127 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 128 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 129 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 130 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 131 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 132 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 133 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 134 #endif 135 } 136 137 static void 138 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 139 { 140 struct cesa_dma_mem *cdm; 141 142 if (error) 143 return; 144 145 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 146 cdm = arg; 147 cdm->cdm_paddr = segs->ds_addr; 148 } 149 150 static int 151 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 152 bus_size_t size) 153 { 154 int error; 155 156 KASSERT(cdm->cdm_vaddr == NULL, 157 ("%s(): DMA memory descriptor in use.", __func__)); 158 159 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 160 PAGE_SIZE, 0, /* alignment, boundary */ 161 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 162 BUS_SPACE_MAXADDR, /* highaddr */ 163 NULL, NULL, /* filtfunc, filtfuncarg */ 164 size, 1, /* maxsize, nsegments */ 165 size, 0, /* maxsegsz, flags */ 166 NULL, NULL, /* lockfunc, lockfuncarg */ 167 &cdm->cdm_tag); /* dmat */ 168 if (error) { 169 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 170 " %i!\n", error); 171 172 goto err1; 173 } 174 175 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 176 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 177 if (error) { 178 device_printf(sc->sc_dev, "failed to allocate DMA safe" 179 " memory, error %i!\n", error); 180 181 goto err2; 182 } 183 184 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 185 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 186 if (error) { 187 device_printf(sc->sc_dev, "cannot get address of the DMA" 188 " memory, error %i\n", error); 189 190 goto err3; 191 } 192 193 return (0); 194 err3: 195 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 196 err2: 197 bus_dma_tag_destroy(cdm->cdm_tag); 198 err1: 199 cdm->cdm_vaddr = NULL; 200 return (error); 201 } 202 203 static void 204 cesa_free_dma_mem(struct cesa_dma_mem *cdm) 205 { 206 207 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 208 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 209 bus_dma_tag_destroy(cdm->cdm_tag); 210 cdm->cdm_vaddr = NULL; 211 } 212 213 static void 214 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 215 { 216 217 /* Sync only if dma memory is valid */ 218 if (cdm->cdm_vaddr != NULL) 219 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 220 } 221 222 static void 223 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 224 { 225 226 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 227 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 228 cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 229 } 230 231 static struct cesa_session * 232 cesa_alloc_session(struct cesa_softc *sc) 233 { 234 struct cesa_session *cs; 235 236 CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions); 237 238 return (cs); 239 } 240 241 static struct cesa_session * 242 cesa_get_session(struct cesa_softc *sc, uint32_t sid) 243 { 244 245 if (sid >= CESA_SESSIONS) 246 return (NULL); 247 248 return (&sc->sc_sessions[sid]); 249 } 250 251 static void 252 cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs) 253 { 254 255 CESA_GENERIC_FREE_LOCKED(sc, cs, sessions); 256 } 257 258 static struct cesa_request * 259 cesa_alloc_request(struct cesa_softc *sc) 260 { 261 struct cesa_request *cr; 262 263 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 264 if (!cr) 265 return (NULL); 266 267 STAILQ_INIT(&cr->cr_tdesc); 268 STAILQ_INIT(&cr->cr_sdesc); 269 270 return (cr); 271 } 272 273 static void 274 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 275 { 276 277 /* Free TDMA descriptors assigned to this request */ 278 CESA_LOCK(sc, tdesc); 279 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 280 CESA_UNLOCK(sc, tdesc); 281 282 /* Free SA descriptors assigned to this request */ 283 CESA_LOCK(sc, sdesc); 284 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 285 CESA_UNLOCK(sc, sdesc); 286 287 /* Unload DMA memory asociated with request */ 288 if (cr->cr_dmap_loaded) { 289 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 290 cr->cr_dmap_loaded = 0; 291 } 292 293 CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 294 } 295 296 static void 297 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 298 { 299 300 CESA_LOCK(sc, requests); 301 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 302 CESA_UNLOCK(sc, requests); 303 } 304 305 static struct cesa_tdma_desc * 306 cesa_alloc_tdesc(struct cesa_softc *sc) 307 { 308 struct cesa_tdma_desc *ctd; 309 310 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 311 312 if (!ctd) 313 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 314 "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 315 316 return (ctd); 317 } 318 319 static struct cesa_sa_desc * 320 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 321 { 322 struct cesa_sa_desc *csd; 323 324 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 325 if (!csd) { 326 device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 327 "Consider increasing CESA_SA_DESCRIPTORS.\n"); 328 return (NULL); 329 } 330 331 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 332 333 /* Fill-in SA descriptor with default values */ 334 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 335 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 336 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 337 csd->csd_cshd->cshd_enc_src = 0; 338 csd->csd_cshd->cshd_enc_dst = 0; 339 csd->csd_cshd->cshd_enc_dlen = 0; 340 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 341 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 342 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 343 csd->csd_cshd->cshd_mac_src = 0; 344 csd->csd_cshd->cshd_mac_dlen = 0; 345 346 return (csd); 347 } 348 349 static struct cesa_tdma_desc * 350 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 351 bus_size_t size) 352 { 353 struct cesa_tdma_desc *ctd; 354 355 ctd = cesa_alloc_tdesc(sc); 356 if (!ctd) 357 return (NULL); 358 359 ctd->ctd_cthd->cthd_dst = dst; 360 ctd->ctd_cthd->cthd_src = src; 361 ctd->ctd_cthd->cthd_byte_count = size; 362 363 /* Handle special control packet */ 364 if (size != 0) 365 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 366 else 367 ctd->ctd_cthd->cthd_flags = 0; 368 369 return (ctd); 370 } 371 372 static struct cesa_tdma_desc * 373 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 374 { 375 376 return (cesa_tdma_copy(sc, sc->sc_sram_base + 377 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 378 sizeof(struct cesa_sa_data))); 379 } 380 381 static struct cesa_tdma_desc * 382 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 383 { 384 385 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base + 386 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 387 } 388 389 static struct cesa_tdma_desc * 390 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 391 { 392 393 return (cesa_tdma_copy(sc, sc->sc_sram_base, csd->csd_cshd_paddr, 394 sizeof(struct cesa_sa_hdesc))); 395 } 396 397 static void 398 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 399 { 400 struct cesa_tdma_desc *ctd_prev; 401 402 if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 403 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 404 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 405 } 406 407 ctd->ctd_cthd->cthd_next = 0; 408 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 409 } 410 411 static int 412 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 413 struct cesa_packet *cp, struct cesa_sa_desc *csd) 414 { 415 struct cesa_tdma_desc *ctd, *tmp; 416 417 /* Copy SA descriptor for this packet */ 418 ctd = cesa_tdma_copy_sdesc(sc, csd); 419 if (!ctd) 420 return (ENOMEM); 421 422 cesa_append_tdesc(cr, ctd); 423 424 /* Copy data to be processed */ 425 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 426 cesa_append_tdesc(cr, ctd); 427 STAILQ_INIT(&cp->cp_copyin); 428 429 /* Insert control descriptor */ 430 ctd = cesa_tdma_copy(sc, 0, 0, 0); 431 if (!ctd) 432 return (ENOMEM); 433 434 cesa_append_tdesc(cr, ctd); 435 436 /* Copy back results */ 437 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 438 cesa_append_tdesc(cr, ctd); 439 STAILQ_INIT(&cp->cp_copyout); 440 441 return (0); 442 } 443 444 static int 445 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 446 { 447 uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; 448 uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; 449 SHA1_CTX sha1ctx; 450 MD5_CTX md5ctx; 451 uint32_t *hout; 452 uint32_t *hin; 453 int i; 454 455 memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 456 memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 457 for (i = 0; i < mklen; i++) { 458 ipad[i] ^= mkey[i]; 459 opad[i] ^= mkey[i]; 460 } 461 462 hin = (uint32_t *)cs->cs_hiv_in; 463 hout = (uint32_t *)cs->cs_hiv_out; 464 465 switch (alg) { 466 case CRYPTO_MD5_HMAC: 467 MD5Init(&md5ctx); 468 MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN); 469 memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); 470 MD5Init(&md5ctx); 471 MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN); 472 memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); 473 break; 474 case CRYPTO_SHA1_HMAC: 475 SHA1Init(&sha1ctx); 476 SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN); 477 memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 478 SHA1Init(&sha1ctx); 479 SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN); 480 memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 481 break; 482 default: 483 return (EINVAL); 484 } 485 486 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 487 hin[i] = htobe32(hin[i]); 488 hout[i] = htobe32(hout[i]); 489 } 490 491 return (0); 492 } 493 494 static int 495 cesa_prep_aes_key(struct cesa_session *cs) 496 { 497 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 498 uint32_t *dkey; 499 int i; 500 501 rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); 502 503 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 504 dkey = (uint32_t *)cs->cs_aes_dkey; 505 506 switch (cs->cs_klen) { 507 case 16: 508 cs->cs_config |= CESA_CSH_AES_KLEN_128; 509 for (i = 0; i < 4; i++) 510 *dkey++ = htobe32(ek[4 * 10 + i]); 511 break; 512 case 24: 513 cs->cs_config |= CESA_CSH_AES_KLEN_192; 514 for (i = 0; i < 4; i++) 515 *dkey++ = htobe32(ek[4 * 12 + i]); 516 for (i = 0; i < 2; i++) 517 *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 518 break; 519 case 32: 520 cs->cs_config |= CESA_CSH_AES_KLEN_256; 521 for (i = 0; i < 4; i++) 522 *dkey++ = htobe32(ek[4 * 14 + i]); 523 for (i = 0; i < 4; i++) 524 *dkey++ = htobe32(ek[4 * 13 + i]); 525 break; 526 default: 527 return (EINVAL); 528 } 529 530 return (0); 531 } 532 533 static int 534 cesa_is_hash(int alg) 535 { 536 537 switch (alg) { 538 case CRYPTO_MD5: 539 case CRYPTO_MD5_HMAC: 540 case CRYPTO_SHA1: 541 case CRYPTO_SHA1_HMAC: 542 return (1); 543 default: 544 return (0); 545 } 546 } 547 548 static void 549 cesa_start_packet(struct cesa_packet *cp, unsigned int size) 550 { 551 552 cp->cp_size = size; 553 cp->cp_offset = 0; 554 STAILQ_INIT(&cp->cp_copyin); 555 STAILQ_INIT(&cp->cp_copyout); 556 } 557 558 static int 559 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 560 bus_dma_segment_t *seg) 561 { 562 struct cesa_tdma_desc *ctd; 563 unsigned int bsize; 564 565 /* Calculate size of block copy */ 566 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 567 568 if (bsize > 0) { 569 ctd = cesa_tdma_copy(sc, sc->sc_sram_base + 570 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 571 if (!ctd) 572 return (-ENOMEM); 573 574 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 575 576 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base + 577 CESA_DATA(cp->cp_offset), bsize); 578 if (!ctd) 579 return (-ENOMEM); 580 581 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 582 583 seg->ds_len -= bsize; 584 seg->ds_addr += bsize; 585 cp->cp_offset += bsize; 586 } 587 588 return (bsize); 589 } 590 591 static void 592 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 593 { 594 unsigned int mpsize, fragmented; 595 unsigned int mlen, mskip, tmlen; 596 struct cesa_chain_info *cci; 597 unsigned int elen, eskip; 598 unsigned int skip, len; 599 struct cesa_sa_desc *csd; 600 struct cesa_request *cr; 601 struct cesa_softc *sc; 602 struct cesa_packet cp; 603 bus_dma_segment_t seg; 604 uint32_t config; 605 int size; 606 607 cci = arg; 608 sc = cci->cci_sc; 609 cr = cci->cci_cr; 610 611 if (error) { 612 cci->cci_error = error; 613 return; 614 } 615 616 elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; 617 eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; 618 mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; 619 mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; 620 621 if (elen && mlen && 622 ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || 623 (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || 624 (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { 625 /* 626 * Data alignment in the request does not meet CESA requiremnts 627 * for combined encryption/decryption and hashing. We have to 628 * split the request to separate operations and process them 629 * one by one. 630 */ 631 config = cci->cci_config; 632 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 633 config &= ~CESA_CSHD_OP_MASK; 634 635 cci->cci_config = config | CESA_CSHD_MAC; 636 cci->cci_enc = NULL; 637 cci->cci_mac = cr->cr_mac; 638 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 639 640 cci->cci_config = config | CESA_CSHD_ENC; 641 cci->cci_enc = cr->cr_enc; 642 cci->cci_mac = NULL; 643 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 644 } else { 645 config &= ~CESA_CSHD_OP_MASK; 646 647 cci->cci_config = config | CESA_CSHD_ENC; 648 cci->cci_enc = cr->cr_enc; 649 cci->cci_mac = NULL; 650 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 651 652 cci->cci_config = config | CESA_CSHD_MAC; 653 cci->cci_enc = NULL; 654 cci->cci_mac = cr->cr_mac; 655 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 656 } 657 658 return; 659 } 660 661 tmlen = mlen; 662 fragmented = 0; 663 mpsize = CESA_MAX_PACKET_SIZE; 664 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 665 666 if (elen && mlen) { 667 skip = MIN(eskip, mskip); 668 len = MAX(elen + eskip, mlen + mskip) - skip; 669 } else if (elen) { 670 skip = eskip; 671 len = elen; 672 } else { 673 skip = mskip; 674 len = mlen; 675 } 676 677 /* Start first packet in chain */ 678 cesa_start_packet(&cp, MIN(mpsize, len)); 679 680 while (nseg-- && len > 0) { 681 seg = *(segs++); 682 683 /* 684 * Skip data in buffer on which neither ENC nor MAC operation 685 * is requested. 686 */ 687 if (skip > 0) { 688 size = MIN(skip, seg.ds_len); 689 skip -= size; 690 691 seg.ds_addr += size; 692 seg.ds_len -= size; 693 694 if (eskip > 0) 695 eskip -= size; 696 697 if (mskip > 0) 698 mskip -= size; 699 700 if (seg.ds_len == 0) 701 continue; 702 } 703 704 while (1) { 705 /* 706 * Fill in current packet with data. Break if there is 707 * no more data in current DMA segment or an error 708 * occured. 709 */ 710 size = cesa_fill_packet(sc, &cp, &seg); 711 if (size <= 0) { 712 error = -size; 713 break; 714 } 715 716 len -= size; 717 718 /* If packet is full, append it to the chain */ 719 if (cp.cp_size == cp.cp_offset) { 720 csd = cesa_alloc_sdesc(sc, cr); 721 if (!csd) { 722 error = ENOMEM; 723 break; 724 } 725 726 /* Create SA descriptor for this packet */ 727 csd->csd_cshd->cshd_config = cci->cci_config; 728 csd->csd_cshd->cshd_mac_total_dlen = tmlen; 729 730 /* 731 * Enable fragmentation if request will not fit 732 * into one packet. 733 */ 734 if (len > 0) { 735 if (!fragmented) { 736 fragmented = 1; 737 csd->csd_cshd->cshd_config |= 738 CESA_CSHD_FRAG_FIRST; 739 } else 740 csd->csd_cshd->cshd_config |= 741 CESA_CSHD_FRAG_MIDDLE; 742 } else if (fragmented) 743 csd->csd_cshd->cshd_config |= 744 CESA_CSHD_FRAG_LAST; 745 746 if (eskip < cp.cp_size && elen > 0) { 747 csd->csd_cshd->cshd_enc_src = 748 CESA_DATA(eskip); 749 csd->csd_cshd->cshd_enc_dst = 750 CESA_DATA(eskip); 751 csd->csd_cshd->cshd_enc_dlen = 752 MIN(elen, cp.cp_size - eskip); 753 } 754 755 if (mskip < cp.cp_size && mlen > 0) { 756 csd->csd_cshd->cshd_mac_src = 757 CESA_DATA(mskip); 758 csd->csd_cshd->cshd_mac_dlen = 759 MIN(mlen, cp.cp_size - mskip); 760 } 761 762 elen -= csd->csd_cshd->cshd_enc_dlen; 763 eskip -= MIN(eskip, cp.cp_size); 764 mlen -= csd->csd_cshd->cshd_mac_dlen; 765 mskip -= MIN(mskip, cp.cp_size); 766 767 cesa_dump_cshd(sc, csd->csd_cshd); 768 769 /* Append packet to the request */ 770 error = cesa_append_packet(sc, cr, &cp, csd); 771 if (error) 772 break; 773 774 /* Start a new packet, as current is full */ 775 cesa_start_packet(&cp, MIN(mpsize, len)); 776 } 777 } 778 779 if (error) 780 break; 781 } 782 783 if (error) { 784 /* 785 * Move all allocated resources to the request. They will be 786 * freed later. 787 */ 788 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 789 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 790 cci->cci_error = error; 791 } 792 } 793 794 static void 795 cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 796 bus_size_t size, int error) 797 { 798 799 cesa_create_chain_cb(arg, segs, nseg, error); 800 } 801 802 static int 803 cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) 804 { 805 struct cesa_chain_info cci; 806 struct cesa_tdma_desc *ctd; 807 uint32_t config; 808 int error; 809 810 error = 0; 811 CESA_LOCK_ASSERT(sc, sessions); 812 813 /* Create request metadata */ 814 if (cr->cr_enc) { 815 if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && 816 (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 817 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 818 cr->cr_cs->cs_klen); 819 else 820 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 821 cr->cr_cs->cs_klen); 822 } 823 824 if (cr->cr_mac) { 825 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 826 CESA_MAX_HASH_LEN); 827 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 828 CESA_MAX_HASH_LEN); 829 } 830 831 ctd = cesa_tdma_copyin_sa_data(sc, cr); 832 if (!ctd) 833 return (ENOMEM); 834 835 cesa_append_tdesc(cr, ctd); 836 837 /* Prepare SA configuration */ 838 config = cr->cr_cs->cs_config; 839 840 if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 841 config |= CESA_CSHD_DECRYPT; 842 if (cr->cr_enc && !cr->cr_mac) 843 config |= CESA_CSHD_ENC; 844 if (!cr->cr_enc && cr->cr_mac) 845 config |= CESA_CSHD_MAC; 846 if (cr->cr_enc && cr->cr_mac) 847 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 848 CESA_CSHD_ENC_AND_MAC; 849 850 /* Create data packets */ 851 cci.cci_sc = sc; 852 cci.cci_cr = cr; 853 cci.cci_enc = cr->cr_enc; 854 cci.cci_mac = cr->cr_mac; 855 cci.cci_config = config; 856 cci.cci_error = 0; 857 858 if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) 859 error = bus_dmamap_load_uio(sc->sc_data_dtag, 860 cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, 861 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 862 else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) 863 error = bus_dmamap_load_mbuf(sc->sc_data_dtag, 864 cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, 865 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 866 else 867 error = bus_dmamap_load(sc->sc_data_dtag, 868 cr->cr_dmap, cr->cr_crp->crp_buf, 869 cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, 870 BUS_DMA_NOWAIT); 871 872 if (!error) 873 cr->cr_dmap_loaded = 1; 874 875 if (cci.cci_error) 876 error = cci.cci_error; 877 878 if (error) 879 return (error); 880 881 /* Read back request metadata */ 882 ctd = cesa_tdma_copyout_sa_data(sc, cr); 883 if (!ctd) 884 return (ENOMEM); 885 886 cesa_append_tdesc(cr, ctd); 887 888 return (0); 889 } 890 891 static void 892 cesa_execute(struct cesa_softc *sc) 893 { 894 struct cesa_tdma_desc *prev_ctd, *ctd; 895 struct cesa_request *prev_cr, *cr; 896 897 CESA_LOCK(sc, requests); 898 899 /* 900 * If ready list is empty, there is nothing to execute. If queued list 901 * is not empty, the hardware is busy and we cannot start another 902 * execution. 903 */ 904 if (STAILQ_EMPTY(&sc->sc_ready_requests) || 905 !STAILQ_EMPTY(&sc->sc_queued_requests)) { 906 CESA_UNLOCK(sc, requests); 907 return; 908 } 909 910 /* Move all ready requests to queued list */ 911 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 912 STAILQ_INIT(&sc->sc_ready_requests); 913 914 /* Create one execution chain from all requests on the list */ 915 if (STAILQ_FIRST(&sc->sc_queued_requests) != 916 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 917 prev_cr = NULL; 918 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 919 BUS_DMASYNC_POSTWRITE); 920 921 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 922 if (prev_cr) { 923 ctd = STAILQ_FIRST(&cr->cr_tdesc); 924 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 925 cesa_tdma_desc, ctd_stq); 926 927 prev_ctd->ctd_cthd->cthd_next = 928 ctd->ctd_cthd_paddr; 929 } 930 931 prev_cr = cr; 932 } 933 934 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 935 BUS_DMASYNC_PREWRITE); 936 } 937 938 /* Start chain execution in hardware */ 939 cr = STAILQ_FIRST(&sc->sc_queued_requests); 940 ctd = STAILQ_FIRST(&cr->cr_tdesc); 941 942 CESA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 943 CESA_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 944 945 CESA_UNLOCK(sc, requests); 946 } 947 948 static int 949 cesa_setup_sram(struct cesa_softc *sc) 950 { 951 phandle_t sram_node; 952 ihandle_t sram_ihandle; 953 pcell_t sram_handle, sram_reg; 954 955 if (OF_getprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 956 (void *)&sram_handle, sizeof(sram_handle)) <= 0) 957 return (ENXIO); 958 959 sram_ihandle = (ihandle_t)sram_handle; 960 sram_ihandle = fdt32_to_cpu(sram_ihandle); 961 sram_node = OF_instance_to_package(sram_ihandle); 962 963 if (OF_getprop(sram_node, "reg", (void *)&sram_reg, 964 sizeof(sram_reg)) <= 0) 965 return (ENXIO); 966 967 sc->sc_sram_base = fdt32_to_cpu(sram_reg); 968 969 return (0); 970 } 971 972 static int 973 cesa_probe(device_t dev) 974 { 975 if (!ofw_bus_is_compatible(dev, "mrvl,cesa")) 976 return (ENXIO); 977 978 device_set_desc(dev, "Marvell Cryptographic Engine and Security " 979 "Accelerator"); 980 981 return (BUS_PROBE_DEFAULT); 982 } 983 984 static int 985 cesa_attach(device_t dev) 986 { 987 struct cesa_softc *sc; 988 uint32_t d, r; 989 int error; 990 int i; 991 992 sc = device_get_softc(dev); 993 sc->sc_blocked = 0; 994 sc->sc_error = 0; 995 sc->sc_dev = dev; 996 997 /* Check if CESA peripheral device has power turned on */ 998 #if defined(SOC_MV_KIRKWOOD) 999 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) == CPU_PM_CTRL_CRYPTO) { 1000 device_printf(dev, "not powered on\n"); 1001 return (ENXIO); 1002 } 1003 #else 1004 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) != CPU_PM_CTRL_CRYPTO) { 1005 device_printf(dev, "not powered on\n"); 1006 return (ENXIO); 1007 } 1008 #endif 1009 soc_id(&d, &r); 1010 1011 switch (d) { 1012 case MV_DEV_88F6281: 1013 case MV_DEV_88F6282: 1014 sc->sc_tperr = 0; 1015 break; 1016 case MV_DEV_MV78100: 1017 case MV_DEV_MV78100_Z0: 1018 sc->sc_tperr = CESA_ICR_TPERR; 1019 break; 1020 default: 1021 return (ENXIO); 1022 } 1023 1024 /* Initialize mutexes */ 1025 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1026 "CESA Shared Data", MTX_DEF); 1027 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1028 "CESA TDMA Descriptors Pool", MTX_DEF); 1029 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1030 "CESA SA Descriptors Pool", MTX_DEF); 1031 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1032 "CESA Requests Pool", MTX_DEF); 1033 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1034 "CESA Sessions Pool", MTX_DEF); 1035 1036 /* Allocate I/O and IRQ resources */ 1037 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1038 if (error) { 1039 device_printf(dev, "could not allocate resources\n"); 1040 goto err0; 1041 } 1042 1043 sc->sc_bsh = rman_get_bushandle(*(sc->sc_res)); 1044 sc->sc_bst = rman_get_bustag(*(sc->sc_res)); 1045 1046 /* Setup CESA decoding windows */ 1047 error = decode_win_cesa_setup(sc); 1048 if (error) { 1049 device_printf(dev, "could not setup decoding windows\n"); 1050 goto err1; 1051 } 1052 1053 /* Acquire SRAM base address */ 1054 error = cesa_setup_sram(sc); 1055 if (error) { 1056 device_printf(dev, "could not setup SRAM\n"); 1057 goto err1; 1058 } 1059 1060 /* Setup interrupt handler */ 1061 error = bus_setup_intr(dev, sc->sc_res[1], INTR_TYPE_NET | INTR_MPSAFE, 1062 NULL, cesa_intr, sc, &(sc->sc_icookie)); 1063 if (error) { 1064 device_printf(dev, "could not setup engine completion irq\n"); 1065 goto err1; 1066 } 1067 1068 /* Create DMA tag for processed data */ 1069 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1070 1, 0, /* alignment, boundary */ 1071 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1072 BUS_SPACE_MAXADDR, /* highaddr */ 1073 NULL, NULL, /* filtfunc, filtfuncarg */ 1074 CESA_MAX_REQUEST_SIZE, /* maxsize */ 1075 CESA_MAX_FRAGMENTS, /* nsegments */ 1076 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1077 NULL, NULL, /* lockfunc, lockfuncarg */ 1078 &sc->sc_data_dtag); /* dmat */ 1079 if (error) 1080 goto err2; 1081 1082 /* Initialize data structures: TDMA Descriptors Pool */ 1083 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1084 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1085 if (error) 1086 goto err3; 1087 1088 STAILQ_INIT(&sc->sc_free_tdesc); 1089 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1090 sc->sc_tdesc[i].ctd_cthd = 1091 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1092 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1093 (i * sizeof(struct cesa_tdma_hdesc)); 1094 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1095 ctd_stq); 1096 } 1097 1098 /* Initialize data structures: SA Descriptors Pool */ 1099 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1100 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1101 if (error) 1102 goto err4; 1103 1104 STAILQ_INIT(&sc->sc_free_sdesc); 1105 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1106 sc->sc_sdesc[i].csd_cshd = 1107 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1108 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1109 (i * sizeof(struct cesa_sa_hdesc)); 1110 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1111 csd_stq); 1112 } 1113 1114 /* Initialize data structures: Requests Pool */ 1115 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1116 CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1117 if (error) 1118 goto err5; 1119 1120 STAILQ_INIT(&sc->sc_free_requests); 1121 STAILQ_INIT(&sc->sc_ready_requests); 1122 STAILQ_INIT(&sc->sc_queued_requests); 1123 for (i = 0; i < CESA_REQUESTS; i++) { 1124 sc->sc_requests[i].cr_csd = 1125 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1126 sc->sc_requests[i].cr_csd_paddr = 1127 sc->sc_requests_cdm.cdm_paddr + 1128 (i * sizeof(struct cesa_sa_data)); 1129 1130 /* Preallocate DMA maps */ 1131 error = bus_dmamap_create(sc->sc_data_dtag, 0, 1132 &sc->sc_requests[i].cr_dmap); 1133 if (error && i > 0) { 1134 i--; 1135 do { 1136 bus_dmamap_destroy(sc->sc_data_dtag, 1137 sc->sc_requests[i].cr_dmap); 1138 } while (i--); 1139 1140 goto err6; 1141 } 1142 1143 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1144 cr_stq); 1145 } 1146 1147 /* Initialize data structures: Sessions Pool */ 1148 STAILQ_INIT(&sc->sc_free_sessions); 1149 for (i = 0; i < CESA_SESSIONS; i++) { 1150 sc->sc_sessions[i].cs_sid = i; 1151 STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i], 1152 cs_stq); 1153 } 1154 1155 /* 1156 * Initialize TDMA: 1157 * - Burst limit: 128 bytes, 1158 * - Outstanding reads enabled, 1159 * - No byte-swap. 1160 */ 1161 CESA_WRITE(sc, CESA_TDMA_CR, CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 | 1162 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE); 1163 1164 /* 1165 * Initialize SA: 1166 * - SA descriptor is present at beginning of CESA SRAM, 1167 * - Multi-packet chain mode, 1168 * - Cooperation with TDMA enabled. 1169 */ 1170 CESA_WRITE(sc, CESA_SA_DPR, 0); 1171 CESA_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1172 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1173 1174 /* Unmask interrupts */ 1175 CESA_WRITE(sc, CESA_ICR, 0); 1176 CESA_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1177 CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1178 CESA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1179 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1180 CESA_TDMA_EMR_DATA_ERROR); 1181 1182 /* Register in OCF */ 1183 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 1184 if (sc->sc_cid) { 1185 device_printf(dev, "could not get crypto driver id\n"); 1186 goto err7; 1187 } 1188 1189 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 1190 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 1191 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 1192 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 1193 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 1194 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 1195 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 1196 1197 return (0); 1198 err7: 1199 for (i = 0; i < CESA_REQUESTS; i++) 1200 bus_dmamap_destroy(sc->sc_data_dtag, 1201 sc->sc_requests[i].cr_dmap); 1202 err6: 1203 cesa_free_dma_mem(&sc->sc_requests_cdm); 1204 err5: 1205 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1206 err4: 1207 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1208 err3: 1209 bus_dma_tag_destroy(sc->sc_data_dtag); 1210 err2: 1211 bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1212 err1: 1213 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1214 err0: 1215 mtx_destroy(&sc->sc_sessions_lock); 1216 mtx_destroy(&sc->sc_requests_lock); 1217 mtx_destroy(&sc->sc_sdesc_lock); 1218 mtx_destroy(&sc->sc_tdesc_lock); 1219 mtx_destroy(&sc->sc_sc_lock); 1220 return (ENXIO); 1221 } 1222 1223 static int 1224 cesa_detach(device_t dev) 1225 { 1226 struct cesa_softc *sc; 1227 int i; 1228 1229 sc = device_get_softc(dev); 1230 1231 /* TODO: Wait for queued requests completion before shutdown. */ 1232 1233 /* Mask interrupts */ 1234 CESA_WRITE(sc, CESA_ICM, 0); 1235 CESA_WRITE(sc, CESA_TDMA_EMR, 0); 1236 1237 /* Unregister from OCF */ 1238 crypto_unregister_all(sc->sc_cid); 1239 1240 /* Free DMA Maps */ 1241 for (i = 0; i < CESA_REQUESTS; i++) 1242 bus_dmamap_destroy(sc->sc_data_dtag, 1243 sc->sc_requests[i].cr_dmap); 1244 1245 /* Free DMA Memory */ 1246 cesa_free_dma_mem(&sc->sc_requests_cdm); 1247 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1248 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1249 1250 /* Free DMA Tag */ 1251 bus_dma_tag_destroy(sc->sc_data_dtag); 1252 1253 /* Stop interrupt */ 1254 bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie); 1255 1256 /* Relase I/O and IRQ resources */ 1257 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1258 1259 /* Destory mutexes */ 1260 mtx_destroy(&sc->sc_sessions_lock); 1261 mtx_destroy(&sc->sc_requests_lock); 1262 mtx_destroy(&sc->sc_sdesc_lock); 1263 mtx_destroy(&sc->sc_tdesc_lock); 1264 mtx_destroy(&sc->sc_sc_lock); 1265 1266 return (0); 1267 } 1268 1269 static void 1270 cesa_intr(void *arg) 1271 { 1272 STAILQ_HEAD(, cesa_request) requests; 1273 struct cesa_request *cr, *tmp; 1274 struct cesa_softc *sc; 1275 uint32_t ecr, icr; 1276 int blocked; 1277 1278 sc = arg; 1279 1280 /* Ack interrupt */ 1281 ecr = CESA_READ(sc, CESA_TDMA_ECR); 1282 CESA_WRITE(sc, CESA_TDMA_ECR, 0); 1283 icr = CESA_READ(sc, CESA_ICR); 1284 CESA_WRITE(sc, CESA_ICR, 0); 1285 1286 /* Check for TDMA errors */ 1287 if (ecr & CESA_TDMA_ECR_MISS) { 1288 device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1289 sc->sc_error = EIO; 1290 } 1291 1292 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1293 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1294 sc->sc_error = EIO; 1295 } 1296 1297 if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1298 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1299 sc->sc_error = EIO; 1300 } 1301 1302 if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1303 device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1304 sc->sc_error = EIO; 1305 } 1306 1307 /* Check for CESA errors */ 1308 if (icr & sc->sc_tperr) { 1309 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1310 sc->sc_error = EIO; 1311 } 1312 1313 /* If there is nothing more to do, return */ 1314 if ((icr & CESA_ICR_ACCTDMA) == 0) 1315 return; 1316 1317 /* Get all finished requests */ 1318 CESA_LOCK(sc, requests); 1319 STAILQ_INIT(&requests); 1320 STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1321 STAILQ_INIT(&sc->sc_queued_requests); 1322 CESA_UNLOCK(sc, requests); 1323 1324 /* Execute all ready requests */ 1325 cesa_execute(sc); 1326 1327 /* Process completed requests */ 1328 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1329 BUS_DMASYNC_POSTWRITE); 1330 1331 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1332 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1333 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1334 1335 cr->cr_crp->crp_etype = sc->sc_error; 1336 if (cr->cr_mac) 1337 crypto_copyback(cr->cr_crp->crp_flags, 1338 cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, 1339 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1340 1341 crypto_done(cr->cr_crp); 1342 cesa_free_request(sc, cr); 1343 } 1344 1345 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1346 BUS_DMASYNC_PREWRITE); 1347 1348 sc->sc_error = 0; 1349 1350 /* Unblock driver if it ran out of resources */ 1351 CESA_LOCK(sc, sc); 1352 blocked = sc->sc_blocked; 1353 sc->sc_blocked = 0; 1354 CESA_UNLOCK(sc, sc); 1355 1356 if (blocked) 1357 crypto_unblock(sc->sc_cid, blocked); 1358 } 1359 1360 static int 1361 cesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 1362 { 1363 struct cesa_session *cs; 1364 struct cesa_softc *sc; 1365 struct cryptoini *enc; 1366 struct cryptoini *mac; 1367 int error; 1368 1369 sc = device_get_softc(dev); 1370 enc = NULL; 1371 mac = NULL; 1372 error = 0; 1373 1374 /* Check and parse input */ 1375 if (cesa_is_hash(cri->cri_alg)) 1376 mac = cri; 1377 else 1378 enc = cri; 1379 1380 cri = cri->cri_next; 1381 1382 if (cri) { 1383 if (!enc && !cesa_is_hash(cri->cri_alg)) 1384 enc = cri; 1385 1386 if (!mac && cesa_is_hash(cri->cri_alg)) 1387 mac = cri; 1388 1389 if (cri->cri_next || !(enc && mac)) 1390 return (EINVAL); 1391 } 1392 1393 if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || 1394 (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) 1395 return (E2BIG); 1396 1397 /* Allocate session */ 1398 cs = cesa_alloc_session(sc); 1399 if (!cs) 1400 return (ENOMEM); 1401 1402 /* Prepare CESA configuration */ 1403 cs->cs_config = 0; 1404 cs->cs_ivlen = 1; 1405 cs->cs_mblen = 1; 1406 1407 if (enc) { 1408 switch (enc->cri_alg) { 1409 case CRYPTO_AES_CBC: 1410 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1411 cs->cs_ivlen = AES_BLOCK_LEN; 1412 break; 1413 case CRYPTO_DES_CBC: 1414 cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; 1415 cs->cs_ivlen = DES_BLOCK_LEN; 1416 break; 1417 case CRYPTO_3DES_CBC: 1418 cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | 1419 CESA_CSHD_CBC; 1420 cs->cs_ivlen = DES3_BLOCK_LEN; 1421 break; 1422 default: 1423 error = EINVAL; 1424 break; 1425 } 1426 } 1427 1428 if (!error && mac) { 1429 switch (mac->cri_alg) { 1430 case CRYPTO_MD5: 1431 cs->cs_config |= CESA_CSHD_MD5; 1432 cs->cs_mblen = 1; 1433 cs->cs_hlen = MD5_HASH_LEN; 1434 break; 1435 case CRYPTO_MD5_HMAC: 1436 cs->cs_config |= CESA_CSHD_MD5_HMAC; 1437 cs->cs_mblen = MD5_HMAC_BLOCK_LEN; 1438 cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1439 break; 1440 case CRYPTO_SHA1: 1441 cs->cs_config |= CESA_CSHD_SHA1; 1442 cs->cs_mblen = 1; 1443 cs->cs_hlen = SHA1_HASH_LEN; 1444 break; 1445 case CRYPTO_SHA1_HMAC: 1446 cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1447 cs->cs_mblen = SHA1_HMAC_BLOCK_LEN; 1448 cs->cs_hlen = CESA_HMAC_HASH_LENGTH; 1449 break; 1450 default: 1451 error = EINVAL; 1452 break; 1453 } 1454 } 1455 1456 /* Save cipher key */ 1457 if (!error && enc && enc->cri_key) { 1458 cs->cs_klen = enc->cri_klen / 8; 1459 memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); 1460 if (enc->cri_alg == CRYPTO_AES_CBC) 1461 error = cesa_prep_aes_key(cs); 1462 } 1463 1464 /* Save digest key */ 1465 if (!error && mac && mac->cri_key) 1466 error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, 1467 mac->cri_klen / 8); 1468 1469 if (error) { 1470 cesa_free_session(sc, cs); 1471 return (EINVAL); 1472 } 1473 1474 *sidp = cs->cs_sid; 1475 1476 return (0); 1477 } 1478 1479 static int 1480 cesa_freesession(device_t dev, uint64_t tid) 1481 { 1482 struct cesa_session *cs; 1483 struct cesa_softc *sc; 1484 1485 sc = device_get_softc(dev); 1486 cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid)); 1487 if (!cs) 1488 return (EINVAL); 1489 1490 /* Free session */ 1491 cesa_free_session(sc, cs); 1492 1493 return (0); 1494 } 1495 1496 static int 1497 cesa_process(device_t dev, struct cryptop *crp, int hint) 1498 { 1499 struct cesa_request *cr; 1500 struct cesa_session *cs; 1501 struct cryptodesc *crd; 1502 struct cryptodesc *enc; 1503 struct cryptodesc *mac; 1504 struct cesa_softc *sc; 1505 int error; 1506 1507 sc = device_get_softc(dev); 1508 crd = crp->crp_desc; 1509 enc = NULL; 1510 mac = NULL; 1511 error = 0; 1512 1513 /* Check session ID */ 1514 cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1515 if (!cs) { 1516 crp->crp_etype = EINVAL; 1517 crypto_done(crp); 1518 return (0); 1519 } 1520 1521 /* Check and parse input */ 1522 if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { 1523 crp->crp_etype = E2BIG; 1524 crypto_done(crp); 1525 return (0); 1526 } 1527 1528 if (cesa_is_hash(crd->crd_alg)) 1529 mac = crd; 1530 else 1531 enc = crd; 1532 1533 crd = crd->crd_next; 1534 1535 if (crd) { 1536 if (!enc && !cesa_is_hash(crd->crd_alg)) 1537 enc = crd; 1538 1539 if (!mac && cesa_is_hash(crd->crd_alg)) 1540 mac = crd; 1541 1542 if (crd->crd_next || !(enc && mac)) { 1543 crp->crp_etype = EINVAL; 1544 crypto_done(crp); 1545 return (0); 1546 } 1547 } 1548 1549 /* 1550 * Get request descriptor. Block driver if there is no free 1551 * descriptors in pool. 1552 */ 1553 cr = cesa_alloc_request(sc); 1554 if (!cr) { 1555 CESA_LOCK(sc, sc); 1556 sc->sc_blocked = CRYPTO_SYMQ; 1557 CESA_UNLOCK(sc, sc); 1558 return (ERESTART); 1559 } 1560 1561 /* Prepare request */ 1562 cr->cr_crp = crp; 1563 cr->cr_enc = enc; 1564 cr->cr_mac = mac; 1565 cr->cr_cs = cs; 1566 1567 CESA_LOCK(sc, sessions); 1568 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1569 1570 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1571 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1572 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1573 else 1574 arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); 1575 1576 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1577 crypto_copyback(crp->crp_flags, crp->crp_buf, 1578 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1579 } else if (enc) { 1580 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1581 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1582 else 1583 crypto_copydata(crp->crp_flags, crp->crp_buf, 1584 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1585 } 1586 1587 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1588 if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { 1589 cs->cs_klen = enc->crd_klen / 8; 1590 memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); 1591 if (enc->crd_alg == CRYPTO_AES_CBC) 1592 error = cesa_prep_aes_key(cs); 1593 } else 1594 error = E2BIG; 1595 } 1596 1597 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1598 if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) 1599 error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, 1600 mac->crd_klen / 8); 1601 else 1602 error = E2BIG; 1603 } 1604 1605 /* Convert request to chain of TDMA and SA descriptors */ 1606 if (!error) 1607 error = cesa_create_chain(sc, cr); 1608 1609 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1610 CESA_UNLOCK(sc, sessions); 1611 1612 if (error) { 1613 cesa_free_request(sc, cr); 1614 crp->crp_etype = error; 1615 crypto_done(crp); 1616 return (0); 1617 } 1618 1619 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1620 BUS_DMASYNC_PREWRITE); 1621 1622 /* Enqueue request to execution */ 1623 cesa_enqueue_request(sc, cr); 1624 1625 /* Start execution, if we have no more requests in queue */ 1626 if ((hint & CRYPTO_HINT_MORE) == 0) 1627 cesa_execute(sc); 1628 1629 return (0); 1630 } 1631 1632 /* 1633 * Set CESA TDMA decode windows. 1634 */ 1635 static int 1636 decode_win_cesa_setup(struct cesa_softc *sc) 1637 { 1638 struct mem_region availmem_regions[FDT_MEM_REGIONS]; 1639 int availmem_regions_sz; 1640 uint32_t memsize, br, cr, i; 1641 1642 /* Grab physical memory regions information from DTS */ 1643 if (fdt_get_mem_regions(availmem_regions, &availmem_regions_sz, 1644 &memsize) != 0) 1645 return (ENXIO); 1646 1647 if (availmem_regions_sz > MV_WIN_CESA_MAX) { 1648 device_printf(sc->sc_dev, "Too much memory regions, cannot " 1649 " set CESA windows to cover whole DRAM \n"); 1650 return (ENXIO); 1651 } 1652 1653 /* Disable and clear all CESA windows */ 1654 for (i = 0; i < MV_WIN_CESA_MAX; i++) { 1655 CESA_WRITE(sc, MV_WIN_CESA_BASE(i), 0); 1656 CESA_WRITE(sc, MV_WIN_CESA_CTRL(i), 0); 1657 } 1658 1659 /* Fill CESA TDMA decoding windows with information acquired from DTS */ 1660 for (i = 0; i < availmem_regions_sz; i++) { 1661 br = availmem_regions[i].mr_start; 1662 cr = availmem_regions[i].mr_size; 1663 1664 /* Don't add entries with size lower than 64KB */ 1665 if (cr & 0xffff0000) { 1666 cr = (((cr - 1) & 0xffff0000) | 1667 (MV_WIN_DDR_ATTR(i) << MV_WIN_CPU_ATTR_SHIFT) | 1668 (MV_WIN_DDR_TARGET << MV_WIN_CPU_TARGET_SHIFT) | 1669 MV_WIN_CPU_ENABLE_BIT); 1670 CESA_WRITE(sc, MV_WIN_CESA_BASE(i), br); 1671 CESA_WRITE(sc, MV_WIN_CESA_CTRL(i), cr); 1672 } 1673 } 1674 1675 return (0); 1676 } 1677 1678