1 /*- 2 * Copyright (C) 2009-2011 Semihalf. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * CESA SRAM Memory Map: 29 * 30 * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE 31 * | | 32 * | DATA | 33 * | | 34 * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0) 35 * | struct cesa_sa_data | 36 * +------------------------+ 37 * | struct cesa_sa_hdesc | 38 * +------------------------+ <= sc->sc_sram_base_va 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/kernel.h> 49 #include <sys/lock.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/mutex.h> 53 #include <sys/rman.h> 54 55 #include <machine/bus.h> 56 #include <machine/intr.h> 57 #include <machine/resource.h> 58 #include <machine/fdt.h> 59 60 #include <dev/fdt/fdt_common.h> 61 #include <dev/ofw/ofw_bus.h> 62 #include <dev/ofw/ofw_bus_subr.h> 63 64 #include <sys/md5.h> 65 #include <crypto/sha1.h> 66 #include <crypto/sha2/sha256.h> 67 #include <crypto/rijndael/rijndael.h> 68 #include <opencrypto/cryptodev.h> 69 #include "cryptodev_if.h" 70 71 #include <arm/mv/mvreg.h> 72 #include <arm/mv/mvwin.h> 73 #include <arm/mv/mvvar.h> 74 #include "cesa.h" 75 76 static int cesa_probe(device_t); 77 static int cesa_attach(device_t); 78 static int cesa_detach(device_t); 79 static void cesa_intr(void *); 80 static int cesa_newsession(device_t, u_int32_t *, struct cryptoini *); 81 static int cesa_freesession(device_t, u_int64_t); 82 static int cesa_process(device_t, struct cryptop *, int); 83 static int decode_win_cesa_setup(struct cesa_softc *sc); 84 85 static struct resource_spec cesa_res_spec[] = { 86 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 87 { SYS_RES_MEMORY, 1, RF_ACTIVE }, 88 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 89 { -1, 0 } 90 }; 91 92 static device_method_t cesa_methods[] = { 93 /* Device interface */ 94 DEVMETHOD(device_probe, cesa_probe), 95 DEVMETHOD(device_attach, cesa_attach), 96 DEVMETHOD(device_detach, cesa_detach), 97 98 /* Crypto device methods */ 99 DEVMETHOD(cryptodev_newsession, cesa_newsession), 100 DEVMETHOD(cryptodev_freesession,cesa_freesession), 101 DEVMETHOD(cryptodev_process, cesa_process), 102 103 DEVMETHOD_END 104 }; 105 106 static driver_t cesa_driver = { 107 "cesa", 108 cesa_methods, 109 sizeof (struct cesa_softc) 110 }; 111 static devclass_t cesa_devclass; 112 113 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); 114 MODULE_DEPEND(cesa, crypto, 1, 1, 1); 115 116 static void 117 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 118 { 119 #ifdef DEBUG 120 device_t dev; 121 122 dev = sc->sc_dev; 123 device_printf(dev, "CESA SA Hardware Descriptor:\n"); 124 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 125 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 126 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 127 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 128 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 129 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 130 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 131 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 132 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 133 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 134 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 135 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 136 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 137 #endif 138 } 139 140 static void 141 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 142 { 143 struct cesa_dma_mem *cdm; 144 145 if (error) 146 return; 147 148 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 149 cdm = arg; 150 cdm->cdm_paddr = segs->ds_addr; 151 } 152 153 static int 154 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 155 bus_size_t size) 156 { 157 int error; 158 159 KASSERT(cdm->cdm_vaddr == NULL, 160 ("%s(): DMA memory descriptor in use.", __func__)); 161 162 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 163 PAGE_SIZE, 0, /* alignment, boundary */ 164 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 165 BUS_SPACE_MAXADDR, /* highaddr */ 166 NULL, NULL, /* filtfunc, filtfuncarg */ 167 size, 1, /* maxsize, nsegments */ 168 size, 0, /* maxsegsz, flags */ 169 NULL, NULL, /* lockfunc, lockfuncarg */ 170 &cdm->cdm_tag); /* dmat */ 171 if (error) { 172 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 173 " %i!\n", error); 174 175 goto err1; 176 } 177 178 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 179 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 180 if (error) { 181 device_printf(sc->sc_dev, "failed to allocate DMA safe" 182 " memory, error %i!\n", error); 183 184 goto err2; 185 } 186 187 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 188 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 189 if (error) { 190 device_printf(sc->sc_dev, "cannot get address of the DMA" 191 " memory, error %i\n", error); 192 193 goto err3; 194 } 195 196 return (0); 197 err3: 198 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 199 err2: 200 bus_dma_tag_destroy(cdm->cdm_tag); 201 err1: 202 cdm->cdm_vaddr = NULL; 203 return (error); 204 } 205 206 static void 207 cesa_free_dma_mem(struct cesa_dma_mem *cdm) 208 { 209 210 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 211 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 212 bus_dma_tag_destroy(cdm->cdm_tag); 213 cdm->cdm_vaddr = NULL; 214 } 215 216 static void 217 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 218 { 219 220 /* Sync only if dma memory is valid */ 221 if (cdm->cdm_vaddr != NULL) 222 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 223 } 224 225 static void 226 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 227 { 228 229 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 230 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 231 cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 232 } 233 234 static struct cesa_session * 235 cesa_alloc_session(struct cesa_softc *sc) 236 { 237 struct cesa_session *cs; 238 239 CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions); 240 241 return (cs); 242 } 243 244 static struct cesa_session * 245 cesa_get_session(struct cesa_softc *sc, uint32_t sid) 246 { 247 248 if (sid >= CESA_SESSIONS) 249 return (NULL); 250 251 return (&sc->sc_sessions[sid]); 252 } 253 254 static void 255 cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs) 256 { 257 258 CESA_GENERIC_FREE_LOCKED(sc, cs, sessions); 259 } 260 261 static struct cesa_request * 262 cesa_alloc_request(struct cesa_softc *sc) 263 { 264 struct cesa_request *cr; 265 266 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 267 if (!cr) 268 return (NULL); 269 270 STAILQ_INIT(&cr->cr_tdesc); 271 STAILQ_INIT(&cr->cr_sdesc); 272 273 return (cr); 274 } 275 276 static void 277 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 278 { 279 280 /* Free TDMA descriptors assigned to this request */ 281 CESA_LOCK(sc, tdesc); 282 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 283 CESA_UNLOCK(sc, tdesc); 284 285 /* Free SA descriptors assigned to this request */ 286 CESA_LOCK(sc, sdesc); 287 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 288 CESA_UNLOCK(sc, sdesc); 289 290 /* Unload DMA memory associated with request */ 291 if (cr->cr_dmap_loaded) { 292 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 293 cr->cr_dmap_loaded = 0; 294 } 295 296 CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 297 } 298 299 static void 300 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 301 { 302 303 CESA_LOCK(sc, requests); 304 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 305 CESA_UNLOCK(sc, requests); 306 } 307 308 static struct cesa_tdma_desc * 309 cesa_alloc_tdesc(struct cesa_softc *sc) 310 { 311 struct cesa_tdma_desc *ctd; 312 313 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 314 315 if (!ctd) 316 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 317 "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 318 319 return (ctd); 320 } 321 322 static struct cesa_sa_desc * 323 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 324 { 325 struct cesa_sa_desc *csd; 326 327 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 328 if (!csd) { 329 device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 330 "Consider increasing CESA_SA_DESCRIPTORS.\n"); 331 return (NULL); 332 } 333 334 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 335 336 /* Fill-in SA descriptor with default values */ 337 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 338 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 339 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 340 csd->csd_cshd->cshd_enc_src = 0; 341 csd->csd_cshd->cshd_enc_dst = 0; 342 csd->csd_cshd->cshd_enc_dlen = 0; 343 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 344 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 345 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 346 csd->csd_cshd->cshd_mac_src = 0; 347 csd->csd_cshd->cshd_mac_dlen = 0; 348 349 return (csd); 350 } 351 352 static struct cesa_tdma_desc * 353 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 354 bus_size_t size) 355 { 356 struct cesa_tdma_desc *ctd; 357 358 ctd = cesa_alloc_tdesc(sc); 359 if (!ctd) 360 return (NULL); 361 362 ctd->ctd_cthd->cthd_dst = dst; 363 ctd->ctd_cthd->cthd_src = src; 364 ctd->ctd_cthd->cthd_byte_count = size; 365 366 /* Handle special control packet */ 367 if (size != 0) 368 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 369 else 370 ctd->ctd_cthd->cthd_flags = 0; 371 372 return (ctd); 373 } 374 375 static struct cesa_tdma_desc * 376 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 377 { 378 379 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa + 380 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 381 sizeof(struct cesa_sa_data))); 382 } 383 384 static struct cesa_tdma_desc * 385 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 386 { 387 388 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa + 389 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 390 } 391 392 static struct cesa_tdma_desc * 393 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 394 { 395 396 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr, 397 sizeof(struct cesa_sa_hdesc))); 398 } 399 400 static void 401 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 402 { 403 struct cesa_tdma_desc *ctd_prev; 404 405 if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 406 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 407 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 408 } 409 410 ctd->ctd_cthd->cthd_next = 0; 411 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 412 } 413 414 static int 415 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 416 struct cesa_packet *cp, struct cesa_sa_desc *csd) 417 { 418 struct cesa_tdma_desc *ctd, *tmp; 419 420 /* Copy SA descriptor for this packet */ 421 ctd = cesa_tdma_copy_sdesc(sc, csd); 422 if (!ctd) 423 return (ENOMEM); 424 425 cesa_append_tdesc(cr, ctd); 426 427 /* Copy data to be processed */ 428 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 429 cesa_append_tdesc(cr, ctd); 430 STAILQ_INIT(&cp->cp_copyin); 431 432 /* Insert control descriptor */ 433 ctd = cesa_tdma_copy(sc, 0, 0, 0); 434 if (!ctd) 435 return (ENOMEM); 436 437 cesa_append_tdesc(cr, ctd); 438 439 /* Copy back results */ 440 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 441 cesa_append_tdesc(cr, ctd); 442 STAILQ_INIT(&cp->cp_copyout); 443 444 return (0); 445 } 446 447 static int 448 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 449 { 450 uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; 451 uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; 452 SHA1_CTX sha1ctx; 453 SHA256_CTX sha256ctx; 454 MD5_CTX md5ctx; 455 uint32_t *hout; 456 uint32_t *hin; 457 int i; 458 459 memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 460 memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 461 for (i = 0; i < mklen; i++) { 462 ipad[i] ^= mkey[i]; 463 opad[i] ^= mkey[i]; 464 } 465 466 hin = (uint32_t *)cs->cs_hiv_in; 467 hout = (uint32_t *)cs->cs_hiv_out; 468 469 switch (alg) { 470 case CRYPTO_MD5_HMAC: 471 MD5Init(&md5ctx); 472 MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN); 473 memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); 474 MD5Init(&md5ctx); 475 MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN); 476 memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); 477 break; 478 case CRYPTO_SHA1_HMAC: 479 SHA1Init(&sha1ctx); 480 SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN); 481 memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 482 SHA1Init(&sha1ctx); 483 SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN); 484 memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 485 break; 486 case CRYPTO_SHA2_256_HMAC: 487 SHA256_Init(&sha256ctx); 488 SHA256_Update(&sha256ctx, ipad, SHA2_256_HMAC_BLOCK_LEN); 489 memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state)); 490 SHA256_Init(&sha256ctx); 491 SHA256_Update(&sha256ctx, opad, SHA2_256_HMAC_BLOCK_LEN); 492 memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state)); 493 break; 494 default: 495 return (EINVAL); 496 } 497 498 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 499 hin[i] = htobe32(hin[i]); 500 hout[i] = htobe32(hout[i]); 501 } 502 503 return (0); 504 } 505 506 static int 507 cesa_prep_aes_key(struct cesa_session *cs) 508 { 509 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 510 uint32_t *dkey; 511 int i; 512 513 rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); 514 515 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 516 dkey = (uint32_t *)cs->cs_aes_dkey; 517 518 switch (cs->cs_klen) { 519 case 16: 520 cs->cs_config |= CESA_CSH_AES_KLEN_128; 521 for (i = 0; i < 4; i++) 522 *dkey++ = htobe32(ek[4 * 10 + i]); 523 break; 524 case 24: 525 cs->cs_config |= CESA_CSH_AES_KLEN_192; 526 for (i = 0; i < 4; i++) 527 *dkey++ = htobe32(ek[4 * 12 + i]); 528 for (i = 0; i < 2; i++) 529 *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 530 break; 531 case 32: 532 cs->cs_config |= CESA_CSH_AES_KLEN_256; 533 for (i = 0; i < 4; i++) 534 *dkey++ = htobe32(ek[4 * 14 + i]); 535 for (i = 0; i < 4; i++) 536 *dkey++ = htobe32(ek[4 * 13 + i]); 537 break; 538 default: 539 return (EINVAL); 540 } 541 542 return (0); 543 } 544 545 static int 546 cesa_is_hash(int alg) 547 { 548 549 switch (alg) { 550 case CRYPTO_MD5: 551 case CRYPTO_MD5_HMAC: 552 case CRYPTO_SHA1: 553 case CRYPTO_SHA1_HMAC: 554 case CRYPTO_SHA2_256_HMAC: 555 return (1); 556 default: 557 return (0); 558 } 559 } 560 561 static void 562 cesa_start_packet(struct cesa_packet *cp, unsigned int size) 563 { 564 565 cp->cp_size = size; 566 cp->cp_offset = 0; 567 STAILQ_INIT(&cp->cp_copyin); 568 STAILQ_INIT(&cp->cp_copyout); 569 } 570 571 static int 572 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 573 bus_dma_segment_t *seg) 574 { 575 struct cesa_tdma_desc *ctd; 576 unsigned int bsize; 577 578 /* Calculate size of block copy */ 579 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 580 581 if (bsize > 0) { 582 ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa + 583 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 584 if (!ctd) 585 return (-ENOMEM); 586 587 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 588 589 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa + 590 CESA_DATA(cp->cp_offset), bsize); 591 if (!ctd) 592 return (-ENOMEM); 593 594 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 595 596 seg->ds_len -= bsize; 597 seg->ds_addr += bsize; 598 cp->cp_offset += bsize; 599 } 600 601 return (bsize); 602 } 603 604 static void 605 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 606 { 607 unsigned int mpsize, fragmented; 608 unsigned int mlen, mskip, tmlen; 609 struct cesa_chain_info *cci; 610 unsigned int elen, eskip; 611 unsigned int skip, len; 612 struct cesa_sa_desc *csd; 613 struct cesa_request *cr; 614 struct cesa_softc *sc; 615 struct cesa_packet cp; 616 bus_dma_segment_t seg; 617 uint32_t config; 618 int size; 619 620 cci = arg; 621 sc = cci->cci_sc; 622 cr = cci->cci_cr; 623 624 if (error) { 625 cci->cci_error = error; 626 return; 627 } 628 629 elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; 630 eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; 631 mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; 632 mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; 633 634 if (elen && mlen && 635 ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || 636 (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || 637 (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { 638 /* 639 * Data alignment in the request does not meet CESA requiremnts 640 * for combined encryption/decryption and hashing. We have to 641 * split the request to separate operations and process them 642 * one by one. 643 */ 644 config = cci->cci_config; 645 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 646 config &= ~CESA_CSHD_OP_MASK; 647 648 cci->cci_config = config | CESA_CSHD_MAC; 649 cci->cci_enc = NULL; 650 cci->cci_mac = cr->cr_mac; 651 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 652 653 cci->cci_config = config | CESA_CSHD_ENC; 654 cci->cci_enc = cr->cr_enc; 655 cci->cci_mac = NULL; 656 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 657 } else { 658 config &= ~CESA_CSHD_OP_MASK; 659 660 cci->cci_config = config | CESA_CSHD_ENC; 661 cci->cci_enc = cr->cr_enc; 662 cci->cci_mac = NULL; 663 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 664 665 cci->cci_config = config | CESA_CSHD_MAC; 666 cci->cci_enc = NULL; 667 cci->cci_mac = cr->cr_mac; 668 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 669 } 670 671 return; 672 } 673 674 tmlen = mlen; 675 fragmented = 0; 676 mpsize = CESA_MAX_PACKET_SIZE; 677 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 678 679 if (elen && mlen) { 680 skip = MIN(eskip, mskip); 681 len = MAX(elen + eskip, mlen + mskip) - skip; 682 } else if (elen) { 683 skip = eskip; 684 len = elen; 685 } else { 686 skip = mskip; 687 len = mlen; 688 } 689 690 /* Start first packet in chain */ 691 cesa_start_packet(&cp, MIN(mpsize, len)); 692 693 while (nseg-- && len > 0) { 694 seg = *(segs++); 695 696 /* 697 * Skip data in buffer on which neither ENC nor MAC operation 698 * is requested. 699 */ 700 if (skip > 0) { 701 size = MIN(skip, seg.ds_len); 702 skip -= size; 703 704 seg.ds_addr += size; 705 seg.ds_len -= size; 706 707 if (eskip > 0) 708 eskip -= size; 709 710 if (mskip > 0) 711 mskip -= size; 712 713 if (seg.ds_len == 0) 714 continue; 715 } 716 717 while (1) { 718 /* 719 * Fill in current packet with data. Break if there is 720 * no more data in current DMA segment or an error 721 * occurred. 722 */ 723 size = cesa_fill_packet(sc, &cp, &seg); 724 if (size <= 0) { 725 error = -size; 726 break; 727 } 728 729 len -= size; 730 731 /* If packet is full, append it to the chain */ 732 if (cp.cp_size == cp.cp_offset) { 733 csd = cesa_alloc_sdesc(sc, cr); 734 if (!csd) { 735 error = ENOMEM; 736 break; 737 } 738 739 /* Create SA descriptor for this packet */ 740 csd->csd_cshd->cshd_config = cci->cci_config; 741 csd->csd_cshd->cshd_mac_total_dlen = tmlen; 742 743 /* 744 * Enable fragmentation if request will not fit 745 * into one packet. 746 */ 747 if (len > 0) { 748 if (!fragmented) { 749 fragmented = 1; 750 csd->csd_cshd->cshd_config |= 751 CESA_CSHD_FRAG_FIRST; 752 } else 753 csd->csd_cshd->cshd_config |= 754 CESA_CSHD_FRAG_MIDDLE; 755 } else if (fragmented) 756 csd->csd_cshd->cshd_config |= 757 CESA_CSHD_FRAG_LAST; 758 759 if (eskip < cp.cp_size && elen > 0) { 760 csd->csd_cshd->cshd_enc_src = 761 CESA_DATA(eskip); 762 csd->csd_cshd->cshd_enc_dst = 763 CESA_DATA(eskip); 764 csd->csd_cshd->cshd_enc_dlen = 765 MIN(elen, cp.cp_size - eskip); 766 } 767 768 if (mskip < cp.cp_size && mlen > 0) { 769 csd->csd_cshd->cshd_mac_src = 770 CESA_DATA(mskip); 771 csd->csd_cshd->cshd_mac_dlen = 772 MIN(mlen, cp.cp_size - mskip); 773 } 774 775 elen -= csd->csd_cshd->cshd_enc_dlen; 776 eskip -= MIN(eskip, cp.cp_size); 777 mlen -= csd->csd_cshd->cshd_mac_dlen; 778 mskip -= MIN(mskip, cp.cp_size); 779 780 cesa_dump_cshd(sc, csd->csd_cshd); 781 782 /* Append packet to the request */ 783 error = cesa_append_packet(sc, cr, &cp, csd); 784 if (error) 785 break; 786 787 /* Start a new packet, as current is full */ 788 cesa_start_packet(&cp, MIN(mpsize, len)); 789 } 790 } 791 792 if (error) 793 break; 794 } 795 796 if (error) { 797 /* 798 * Move all allocated resources to the request. They will be 799 * freed later. 800 */ 801 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 802 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 803 cci->cci_error = error; 804 } 805 } 806 807 static void 808 cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 809 bus_size_t size, int error) 810 { 811 812 cesa_create_chain_cb(arg, segs, nseg, error); 813 } 814 815 static int 816 cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) 817 { 818 struct cesa_chain_info cci; 819 struct cesa_tdma_desc *ctd; 820 uint32_t config; 821 int error; 822 823 error = 0; 824 CESA_LOCK_ASSERT(sc, sessions); 825 826 /* Create request metadata */ 827 if (cr->cr_enc) { 828 if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && 829 (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 830 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 831 cr->cr_cs->cs_klen); 832 else 833 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 834 cr->cr_cs->cs_klen); 835 } 836 837 if (cr->cr_mac) { 838 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 839 CESA_MAX_HASH_LEN); 840 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 841 CESA_MAX_HASH_LEN); 842 } 843 844 ctd = cesa_tdma_copyin_sa_data(sc, cr); 845 if (!ctd) 846 return (ENOMEM); 847 848 cesa_append_tdesc(cr, ctd); 849 850 /* Prepare SA configuration */ 851 config = cr->cr_cs->cs_config; 852 853 if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 854 config |= CESA_CSHD_DECRYPT; 855 if (cr->cr_enc && !cr->cr_mac) 856 config |= CESA_CSHD_ENC; 857 if (!cr->cr_enc && cr->cr_mac) 858 config |= CESA_CSHD_MAC; 859 if (cr->cr_enc && cr->cr_mac) 860 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 861 CESA_CSHD_ENC_AND_MAC; 862 863 /* Create data packets */ 864 cci.cci_sc = sc; 865 cci.cci_cr = cr; 866 cci.cci_enc = cr->cr_enc; 867 cci.cci_mac = cr->cr_mac; 868 cci.cci_config = config; 869 cci.cci_error = 0; 870 871 if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) 872 error = bus_dmamap_load_uio(sc->sc_data_dtag, 873 cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, 874 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 875 else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) 876 error = bus_dmamap_load_mbuf(sc->sc_data_dtag, 877 cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, 878 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 879 else 880 error = bus_dmamap_load(sc->sc_data_dtag, 881 cr->cr_dmap, cr->cr_crp->crp_buf, 882 cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, 883 BUS_DMA_NOWAIT); 884 885 if (!error) 886 cr->cr_dmap_loaded = 1; 887 888 if (cci.cci_error) 889 error = cci.cci_error; 890 891 if (error) 892 return (error); 893 894 /* Read back request metadata */ 895 ctd = cesa_tdma_copyout_sa_data(sc, cr); 896 if (!ctd) 897 return (ENOMEM); 898 899 cesa_append_tdesc(cr, ctd); 900 901 return (0); 902 } 903 904 static void 905 cesa_execute(struct cesa_softc *sc) 906 { 907 struct cesa_tdma_desc *prev_ctd, *ctd; 908 struct cesa_request *prev_cr, *cr; 909 910 CESA_LOCK(sc, requests); 911 912 /* 913 * If ready list is empty, there is nothing to execute. If queued list 914 * is not empty, the hardware is busy and we cannot start another 915 * execution. 916 */ 917 if (STAILQ_EMPTY(&sc->sc_ready_requests) || 918 !STAILQ_EMPTY(&sc->sc_queued_requests)) { 919 CESA_UNLOCK(sc, requests); 920 return; 921 } 922 923 /* Move all ready requests to queued list */ 924 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 925 STAILQ_INIT(&sc->sc_ready_requests); 926 927 /* Create one execution chain from all requests on the list */ 928 if (STAILQ_FIRST(&sc->sc_queued_requests) != 929 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 930 prev_cr = NULL; 931 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 932 BUS_DMASYNC_POSTWRITE); 933 934 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 935 if (prev_cr) { 936 ctd = STAILQ_FIRST(&cr->cr_tdesc); 937 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 938 cesa_tdma_desc, ctd_stq); 939 940 prev_ctd->ctd_cthd->cthd_next = 941 ctd->ctd_cthd_paddr; 942 } 943 944 prev_cr = cr; 945 } 946 947 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 948 BUS_DMASYNC_PREWRITE); 949 } 950 951 /* Start chain execution in hardware */ 952 cr = STAILQ_FIRST(&sc->sc_queued_requests); 953 ctd = STAILQ_FIRST(&cr->cr_tdesc); 954 955 CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 956 957 if (sc->sc_soc_id == MV_DEV_88F6828 || 958 sc->sc_soc_id == MV_DEV_88F6820 || 959 sc->sc_soc_id == MV_DEV_88F6810) 960 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2); 961 else 962 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 963 964 CESA_UNLOCK(sc, requests); 965 } 966 967 static int 968 cesa_setup_sram(struct cesa_softc *sc) 969 { 970 phandle_t sram_node; 971 ihandle_t sram_ihandle; 972 pcell_t sram_handle, sram_reg[2]; 973 void *sram_va; 974 int rv; 975 976 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 977 (void *)&sram_handle, sizeof(sram_handle)); 978 if (rv <= 0) 979 return (rv); 980 981 sram_ihandle = (ihandle_t)sram_handle; 982 sram_node = OF_instance_to_package(sram_ihandle); 983 984 rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg)); 985 if (rv <= 0) 986 return (rv); 987 988 sc->sc_sram_base_pa = sram_reg[0]; 989 /* Store SRAM size to be able to unmap in detach() */ 990 sc->sc_sram_size = sram_reg[1]; 991 992 if (sc->sc_soc_id != MV_DEV_88F6828 && 993 sc->sc_soc_id != MV_DEV_88F6820 && 994 sc->sc_soc_id != MV_DEV_88F6810) 995 return (0); 996 997 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */ 998 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); 999 if (sram_va == NULL) 1000 return (ENOMEM); 1001 sc->sc_sram_base_va = (vm_offset_t)sram_va; 1002 1003 return (0); 1004 } 1005 1006 static int 1007 cesa_probe(device_t dev) 1008 { 1009 1010 if (!ofw_bus_status_okay(dev)) 1011 return (ENXIO); 1012 1013 if (!ofw_bus_is_compatible(dev, "mrvl,cesa")) 1014 return (ENXIO); 1015 1016 device_set_desc(dev, "Marvell Cryptographic Engine and Security " 1017 "Accelerator"); 1018 1019 return (BUS_PROBE_DEFAULT); 1020 } 1021 1022 static int 1023 cesa_attach(device_t dev) 1024 { 1025 struct cesa_softc *sc; 1026 uint32_t d, r, val; 1027 int error; 1028 int i; 1029 1030 sc = device_get_softc(dev); 1031 sc->sc_blocked = 0; 1032 sc->sc_error = 0; 1033 sc->sc_dev = dev; 1034 1035 soc_id(&d, &r); 1036 1037 switch (d) { 1038 case MV_DEV_88F6281: 1039 case MV_DEV_88F6282: 1040 /* Check if CESA peripheral device has power turned on */ 1041 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) == 1042 CPU_PM_CTRL_CRYPTO) { 1043 device_printf(dev, "not powered on\n"); 1044 return (ENXIO); 1045 } 1046 sc->sc_tperr = 0; 1047 break; 1048 case MV_DEV_88F6828: 1049 case MV_DEV_88F6820: 1050 case MV_DEV_88F6810: 1051 sc->sc_tperr = 0; 1052 break; 1053 case MV_DEV_MV78100: 1054 case MV_DEV_MV78100_Z0: 1055 /* Check if CESA peripheral device has power turned on */ 1056 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) != 1057 CPU_PM_CTRL_CRYPTO) { 1058 device_printf(dev, "not powered on\n"); 1059 return (ENXIO); 1060 } 1061 sc->sc_tperr = CESA_ICR_TPERR; 1062 break; 1063 default: 1064 return (ENXIO); 1065 } 1066 1067 sc->sc_soc_id = d; 1068 1069 /* Initialize mutexes */ 1070 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1071 "CESA Shared Data", MTX_DEF); 1072 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1073 "CESA TDMA Descriptors Pool", MTX_DEF); 1074 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1075 "CESA SA Descriptors Pool", MTX_DEF); 1076 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1077 "CESA Requests Pool", MTX_DEF); 1078 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1079 "CESA Sessions Pool", MTX_DEF); 1080 1081 /* Allocate I/O and IRQ resources */ 1082 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1083 if (error) { 1084 device_printf(dev, "could not allocate resources\n"); 1085 goto err0; 1086 } 1087 1088 /* Setup CESA decoding windows */ 1089 error = decode_win_cesa_setup(sc); 1090 if (error) { 1091 device_printf(dev, "could not setup decoding windows\n"); 1092 goto err1; 1093 } 1094 1095 /* Acquire SRAM base address */ 1096 error = cesa_setup_sram(sc); 1097 if (error) { 1098 device_printf(dev, "could not setup SRAM\n"); 1099 goto err1; 1100 } 1101 1102 /* Setup interrupt handler */ 1103 error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET | 1104 INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie)); 1105 if (error) { 1106 device_printf(dev, "could not setup engine completion irq\n"); 1107 goto err2; 1108 } 1109 1110 /* Create DMA tag for processed data */ 1111 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1112 1, 0, /* alignment, boundary */ 1113 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1114 BUS_SPACE_MAXADDR, /* highaddr */ 1115 NULL, NULL, /* filtfunc, filtfuncarg */ 1116 CESA_MAX_REQUEST_SIZE, /* maxsize */ 1117 CESA_MAX_FRAGMENTS, /* nsegments */ 1118 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1119 NULL, NULL, /* lockfunc, lockfuncarg */ 1120 &sc->sc_data_dtag); /* dmat */ 1121 if (error) 1122 goto err3; 1123 1124 /* Initialize data structures: TDMA Descriptors Pool */ 1125 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1126 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1127 if (error) 1128 goto err4; 1129 1130 STAILQ_INIT(&sc->sc_free_tdesc); 1131 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1132 sc->sc_tdesc[i].ctd_cthd = 1133 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1134 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1135 (i * sizeof(struct cesa_tdma_hdesc)); 1136 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1137 ctd_stq); 1138 } 1139 1140 /* Initialize data structures: SA Descriptors Pool */ 1141 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1142 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1143 if (error) 1144 goto err5; 1145 1146 STAILQ_INIT(&sc->sc_free_sdesc); 1147 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1148 sc->sc_sdesc[i].csd_cshd = 1149 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1150 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1151 (i * sizeof(struct cesa_sa_hdesc)); 1152 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1153 csd_stq); 1154 } 1155 1156 /* Initialize data structures: Requests Pool */ 1157 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1158 CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1159 if (error) 1160 goto err6; 1161 1162 STAILQ_INIT(&sc->sc_free_requests); 1163 STAILQ_INIT(&sc->sc_ready_requests); 1164 STAILQ_INIT(&sc->sc_queued_requests); 1165 for (i = 0; i < CESA_REQUESTS; i++) { 1166 sc->sc_requests[i].cr_csd = 1167 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1168 sc->sc_requests[i].cr_csd_paddr = 1169 sc->sc_requests_cdm.cdm_paddr + 1170 (i * sizeof(struct cesa_sa_data)); 1171 1172 /* Preallocate DMA maps */ 1173 error = bus_dmamap_create(sc->sc_data_dtag, 0, 1174 &sc->sc_requests[i].cr_dmap); 1175 if (error && i > 0) { 1176 i--; 1177 do { 1178 bus_dmamap_destroy(sc->sc_data_dtag, 1179 sc->sc_requests[i].cr_dmap); 1180 } while (i--); 1181 1182 goto err7; 1183 } 1184 1185 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1186 cr_stq); 1187 } 1188 1189 /* Initialize data structures: Sessions Pool */ 1190 STAILQ_INIT(&sc->sc_free_sessions); 1191 for (i = 0; i < CESA_SESSIONS; i++) { 1192 sc->sc_sessions[i].cs_sid = i; 1193 STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i], 1194 cs_stq); 1195 } 1196 1197 /* 1198 * Initialize TDMA: 1199 * - Burst limit: 128 bytes, 1200 * - Outstanding reads enabled, 1201 * - No byte-swap. 1202 */ 1203 val = CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 | 1204 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE; 1205 1206 if (sc->sc_soc_id == MV_DEV_88F6828 || 1207 sc->sc_soc_id == MV_DEV_88F6820 || 1208 sc->sc_soc_id == MV_DEV_88F6810) 1209 val |= CESA_TDMA_NUM_OUTSTAND; 1210 1211 CESA_TDMA_WRITE(sc, CESA_TDMA_CR, val); 1212 1213 /* 1214 * Initialize SA: 1215 * - SA descriptor is present at beginning of CESA SRAM, 1216 * - Multi-packet chain mode, 1217 * - Cooperation with TDMA enabled. 1218 */ 1219 CESA_REG_WRITE(sc, CESA_SA_DPR, 0); 1220 CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1221 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1222 1223 /* Unmask interrupts */ 1224 CESA_REG_WRITE(sc, CESA_ICR, 0); 1225 CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1226 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1227 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1228 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1229 CESA_TDMA_EMR_DATA_ERROR); 1230 1231 /* Register in OCF */ 1232 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 1233 if (sc->sc_cid < 0) { 1234 device_printf(dev, "could not get crypto driver id\n"); 1235 goto err8; 1236 } 1237 1238 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 1239 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 1240 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 1241 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 1242 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 1243 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 1244 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 1245 if (sc->sc_soc_id == MV_DEV_88F6828 || 1246 sc->sc_soc_id == MV_DEV_88F6820 || 1247 sc->sc_soc_id == MV_DEV_88F6810) 1248 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 1249 1250 return (0); 1251 err8: 1252 for (i = 0; i < CESA_REQUESTS; i++) 1253 bus_dmamap_destroy(sc->sc_data_dtag, 1254 sc->sc_requests[i].cr_dmap); 1255 err7: 1256 cesa_free_dma_mem(&sc->sc_requests_cdm); 1257 err6: 1258 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1259 err5: 1260 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1261 err4: 1262 bus_dma_tag_destroy(sc->sc_data_dtag); 1263 err3: 1264 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1265 err2: 1266 if (sc->sc_soc_id == MV_DEV_88F6828 || 1267 sc->sc_soc_id == MV_DEV_88F6820 || 1268 sc->sc_soc_id == MV_DEV_88F6810) 1269 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1270 err1: 1271 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1272 err0: 1273 mtx_destroy(&sc->sc_sessions_lock); 1274 mtx_destroy(&sc->sc_requests_lock); 1275 mtx_destroy(&sc->sc_sdesc_lock); 1276 mtx_destroy(&sc->sc_tdesc_lock); 1277 mtx_destroy(&sc->sc_sc_lock); 1278 return (ENXIO); 1279 } 1280 1281 static int 1282 cesa_detach(device_t dev) 1283 { 1284 struct cesa_softc *sc; 1285 int i; 1286 1287 sc = device_get_softc(dev); 1288 1289 /* TODO: Wait for queued requests completion before shutdown. */ 1290 1291 /* Mask interrupts */ 1292 CESA_REG_WRITE(sc, CESA_ICM, 0); 1293 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0); 1294 1295 /* Unregister from OCF */ 1296 crypto_unregister_all(sc->sc_cid); 1297 1298 /* Free DMA Maps */ 1299 for (i = 0; i < CESA_REQUESTS; i++) 1300 bus_dmamap_destroy(sc->sc_data_dtag, 1301 sc->sc_requests[i].cr_dmap); 1302 1303 /* Free DMA Memory */ 1304 cesa_free_dma_mem(&sc->sc_requests_cdm); 1305 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1306 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1307 1308 /* Free DMA Tag */ 1309 bus_dma_tag_destroy(sc->sc_data_dtag); 1310 1311 /* Stop interrupt */ 1312 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1313 1314 /* Relase I/O and IRQ resources */ 1315 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1316 1317 /* Unmap SRAM memory */ 1318 if (sc->sc_soc_id == MV_DEV_88F6828 || 1319 sc->sc_soc_id == MV_DEV_88F6820 || 1320 sc->sc_soc_id == MV_DEV_88F6810) 1321 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1322 1323 /* Destroy mutexes */ 1324 mtx_destroy(&sc->sc_sessions_lock); 1325 mtx_destroy(&sc->sc_requests_lock); 1326 mtx_destroy(&sc->sc_sdesc_lock); 1327 mtx_destroy(&sc->sc_tdesc_lock); 1328 mtx_destroy(&sc->sc_sc_lock); 1329 1330 return (0); 1331 } 1332 1333 static void 1334 cesa_intr(void *arg) 1335 { 1336 STAILQ_HEAD(, cesa_request) requests; 1337 struct cesa_request *cr, *tmp; 1338 struct cesa_softc *sc; 1339 uint32_t ecr, icr; 1340 int blocked; 1341 1342 sc = arg; 1343 1344 /* Ack interrupt */ 1345 ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR); 1346 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1347 icr = CESA_REG_READ(sc, CESA_ICR); 1348 CESA_REG_WRITE(sc, CESA_ICR, 0); 1349 1350 /* Check for TDMA errors */ 1351 if (ecr & CESA_TDMA_ECR_MISS) { 1352 device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1353 sc->sc_error = EIO; 1354 } 1355 1356 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1357 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1358 sc->sc_error = EIO; 1359 } 1360 1361 if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1362 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1363 sc->sc_error = EIO; 1364 } 1365 1366 if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1367 device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1368 sc->sc_error = EIO; 1369 } 1370 1371 /* Check for CESA errors */ 1372 if (icr & sc->sc_tperr) { 1373 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1374 sc->sc_error = EIO; 1375 } 1376 1377 /* If there is nothing more to do, return */ 1378 if ((icr & CESA_ICR_ACCTDMA) == 0) 1379 return; 1380 1381 /* Get all finished requests */ 1382 CESA_LOCK(sc, requests); 1383 STAILQ_INIT(&requests); 1384 STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1385 STAILQ_INIT(&sc->sc_queued_requests); 1386 CESA_UNLOCK(sc, requests); 1387 1388 /* Execute all ready requests */ 1389 cesa_execute(sc); 1390 1391 /* Process completed requests */ 1392 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1393 BUS_DMASYNC_POSTWRITE); 1394 1395 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1396 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1397 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1398 1399 cr->cr_crp->crp_etype = sc->sc_error; 1400 if (cr->cr_mac) 1401 crypto_copyback(cr->cr_crp->crp_flags, 1402 cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, 1403 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1404 1405 crypto_done(cr->cr_crp); 1406 cesa_free_request(sc, cr); 1407 } 1408 1409 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1410 BUS_DMASYNC_PREWRITE); 1411 1412 sc->sc_error = 0; 1413 1414 /* Unblock driver if it ran out of resources */ 1415 CESA_LOCK(sc, sc); 1416 blocked = sc->sc_blocked; 1417 sc->sc_blocked = 0; 1418 CESA_UNLOCK(sc, sc); 1419 1420 if (blocked) 1421 crypto_unblock(sc->sc_cid, blocked); 1422 } 1423 1424 static int 1425 cesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 1426 { 1427 struct cesa_session *cs; 1428 struct cesa_softc *sc; 1429 struct cryptoini *enc; 1430 struct cryptoini *mac; 1431 int error; 1432 1433 sc = device_get_softc(dev); 1434 enc = NULL; 1435 mac = NULL; 1436 error = 0; 1437 1438 /* Check and parse input */ 1439 if (cesa_is_hash(cri->cri_alg)) 1440 mac = cri; 1441 else 1442 enc = cri; 1443 1444 cri = cri->cri_next; 1445 1446 if (cri) { 1447 if (!enc && !cesa_is_hash(cri->cri_alg)) 1448 enc = cri; 1449 1450 if (!mac && cesa_is_hash(cri->cri_alg)) 1451 mac = cri; 1452 1453 if (cri->cri_next || !(enc && mac)) 1454 return (EINVAL); 1455 } 1456 1457 if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || 1458 (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) 1459 return (E2BIG); 1460 1461 /* Allocate session */ 1462 cs = cesa_alloc_session(sc); 1463 if (!cs) 1464 return (ENOMEM); 1465 1466 /* Prepare CESA configuration */ 1467 cs->cs_config = 0; 1468 cs->cs_ivlen = 1; 1469 cs->cs_mblen = 1; 1470 1471 if (enc) { 1472 switch (enc->cri_alg) { 1473 case CRYPTO_AES_CBC: 1474 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1475 cs->cs_ivlen = AES_BLOCK_LEN; 1476 break; 1477 case CRYPTO_DES_CBC: 1478 cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; 1479 cs->cs_ivlen = DES_BLOCK_LEN; 1480 break; 1481 case CRYPTO_3DES_CBC: 1482 cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | 1483 CESA_CSHD_CBC; 1484 cs->cs_ivlen = DES3_BLOCK_LEN; 1485 break; 1486 default: 1487 error = EINVAL; 1488 break; 1489 } 1490 } 1491 1492 if (!error && mac) { 1493 switch (mac->cri_alg) { 1494 case CRYPTO_MD5: 1495 cs->cs_mblen = 1; 1496 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : 1497 mac->cri_mlen; 1498 cs->cs_config |= CESA_CSHD_MD5; 1499 break; 1500 case CRYPTO_MD5_HMAC: 1501 cs->cs_mblen = MD5_HMAC_BLOCK_LEN; 1502 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : 1503 mac->cri_mlen; 1504 cs->cs_config |= CESA_CSHD_MD5_HMAC; 1505 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) 1506 cs->cs_config |= CESA_CSHD_96_BIT_HMAC; 1507 break; 1508 case CRYPTO_SHA1: 1509 cs->cs_mblen = 1; 1510 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : 1511 mac->cri_mlen; 1512 cs->cs_config |= CESA_CSHD_SHA1; 1513 break; 1514 case CRYPTO_SHA1_HMAC: 1515 cs->cs_mblen = SHA1_HMAC_BLOCK_LEN; 1516 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : 1517 mac->cri_mlen; 1518 cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1519 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) 1520 cs->cs_config |= CESA_CSHD_96_BIT_HMAC; 1521 break; 1522 case CRYPTO_SHA2_256_HMAC: 1523 cs->cs_mblen = SHA2_256_HMAC_BLOCK_LEN; 1524 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN : 1525 mac->cri_mlen; 1526 cs->cs_config |= CESA_CSHD_SHA2_256_HMAC; 1527 break; 1528 default: 1529 error = EINVAL; 1530 break; 1531 } 1532 } 1533 1534 /* Save cipher key */ 1535 if (!error && enc && enc->cri_key) { 1536 cs->cs_klen = enc->cri_klen / 8; 1537 memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); 1538 if (enc->cri_alg == CRYPTO_AES_CBC) 1539 error = cesa_prep_aes_key(cs); 1540 } 1541 1542 /* Save digest key */ 1543 if (!error && mac && mac->cri_key) 1544 error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, 1545 mac->cri_klen / 8); 1546 1547 if (error) { 1548 cesa_free_session(sc, cs); 1549 return (EINVAL); 1550 } 1551 1552 *sidp = cs->cs_sid; 1553 1554 return (0); 1555 } 1556 1557 static int 1558 cesa_freesession(device_t dev, uint64_t tid) 1559 { 1560 struct cesa_session *cs; 1561 struct cesa_softc *sc; 1562 1563 sc = device_get_softc(dev); 1564 cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid)); 1565 if (!cs) 1566 return (EINVAL); 1567 1568 /* Free session */ 1569 cesa_free_session(sc, cs); 1570 1571 return (0); 1572 } 1573 1574 static int 1575 cesa_process(device_t dev, struct cryptop *crp, int hint) 1576 { 1577 struct cesa_request *cr; 1578 struct cesa_session *cs; 1579 struct cryptodesc *crd; 1580 struct cryptodesc *enc; 1581 struct cryptodesc *mac; 1582 struct cesa_softc *sc; 1583 int error; 1584 1585 sc = device_get_softc(dev); 1586 crd = crp->crp_desc; 1587 enc = NULL; 1588 mac = NULL; 1589 error = 0; 1590 1591 /* Check session ID */ 1592 cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 1593 if (!cs) { 1594 crp->crp_etype = EINVAL; 1595 crypto_done(crp); 1596 return (0); 1597 } 1598 1599 /* Check and parse input */ 1600 if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { 1601 crp->crp_etype = E2BIG; 1602 crypto_done(crp); 1603 return (0); 1604 } 1605 1606 if (cesa_is_hash(crd->crd_alg)) 1607 mac = crd; 1608 else 1609 enc = crd; 1610 1611 crd = crd->crd_next; 1612 1613 if (crd) { 1614 if (!enc && !cesa_is_hash(crd->crd_alg)) 1615 enc = crd; 1616 1617 if (!mac && cesa_is_hash(crd->crd_alg)) 1618 mac = crd; 1619 1620 if (crd->crd_next || !(enc && mac)) { 1621 crp->crp_etype = EINVAL; 1622 crypto_done(crp); 1623 return (0); 1624 } 1625 } 1626 1627 /* 1628 * Get request descriptor. Block driver if there is no free 1629 * descriptors in pool. 1630 */ 1631 cr = cesa_alloc_request(sc); 1632 if (!cr) { 1633 CESA_LOCK(sc, sc); 1634 sc->sc_blocked = CRYPTO_SYMQ; 1635 CESA_UNLOCK(sc, sc); 1636 return (ERESTART); 1637 } 1638 1639 /* Prepare request */ 1640 cr->cr_crp = crp; 1641 cr->cr_enc = enc; 1642 cr->cr_mac = mac; 1643 cr->cr_cs = cs; 1644 1645 CESA_LOCK(sc, sessions); 1646 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1647 1648 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1649 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1650 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1651 else 1652 arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); 1653 1654 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1655 crypto_copyback(crp->crp_flags, crp->crp_buf, 1656 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1657 } else if (enc) { 1658 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1659 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1660 else 1661 crypto_copydata(crp->crp_flags, crp->crp_buf, 1662 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1663 } 1664 1665 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1666 if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { 1667 cs->cs_klen = enc->crd_klen / 8; 1668 memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); 1669 if (enc->crd_alg == CRYPTO_AES_CBC) 1670 error = cesa_prep_aes_key(cs); 1671 } else 1672 error = E2BIG; 1673 } 1674 1675 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1676 if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) 1677 error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, 1678 mac->crd_klen / 8); 1679 else 1680 error = E2BIG; 1681 } 1682 1683 /* Convert request to chain of TDMA and SA descriptors */ 1684 if (!error) 1685 error = cesa_create_chain(sc, cr); 1686 1687 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1688 CESA_UNLOCK(sc, sessions); 1689 1690 if (error) { 1691 cesa_free_request(sc, cr); 1692 crp->crp_etype = error; 1693 crypto_done(crp); 1694 return (0); 1695 } 1696 1697 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1698 BUS_DMASYNC_PREWRITE); 1699 1700 /* Enqueue request to execution */ 1701 cesa_enqueue_request(sc, cr); 1702 1703 /* Start execution, if we have no more requests in queue */ 1704 if ((hint & CRYPTO_HINT_MORE) == 0) 1705 cesa_execute(sc); 1706 1707 return (0); 1708 } 1709 1710 /* 1711 * Set CESA TDMA decode windows. 1712 */ 1713 static int 1714 decode_win_cesa_setup(struct cesa_softc *sc) 1715 { 1716 struct mem_region availmem_regions[FDT_MEM_REGIONS]; 1717 int availmem_regions_sz; 1718 uint32_t br, cr, i; 1719 1720 /* Grab physical memory regions information from DTS */ 1721 if (fdt_get_mem_regions(availmem_regions, &availmem_regions_sz, 1722 NULL) != 0) 1723 return (ENXIO); 1724 1725 if (availmem_regions_sz > MV_WIN_CESA_MAX) { 1726 device_printf(sc->sc_dev, "Too much memory regions, cannot " 1727 " set CESA windows to cover whole DRAM \n"); 1728 return (ENXIO); 1729 } 1730 1731 /* Disable and clear all CESA windows */ 1732 for (i = 0; i < MV_WIN_CESA_MAX; i++) { 1733 CESA_TDMA_WRITE(sc, MV_WIN_CESA_BASE(i), 0); 1734 CESA_TDMA_WRITE(sc, MV_WIN_CESA_CTRL(i), 0); 1735 } 1736 1737 /* Fill CESA TDMA decoding windows with information acquired from DTS */ 1738 for (i = 0; i < availmem_regions_sz; i++) { 1739 br = availmem_regions[i].mr_start; 1740 cr = availmem_regions[i].mr_size; 1741 1742 /* Don't add entries with size lower than 64KB */ 1743 if (cr & 0xffff0000) { 1744 cr = (((cr - 1) & 0xffff0000) | 1745 (MV_WIN_DDR_ATTR(i) << MV_WIN_CPU_ATTR_SHIFT) | 1746 (MV_WIN_DDR_TARGET << MV_WIN_CPU_TARGET_SHIFT) | 1747 MV_WIN_CPU_ENABLE_BIT); 1748 CESA_TDMA_WRITE(sc, MV_WIN_CESA_BASE(i), br); 1749 CESA_TDMA_WRITE(sc, MV_WIN_CESA_CTRL(i), cr); 1750 } 1751 } 1752 1753 return (0); 1754 } 1755 1756